2024-12-02 13:17:03,612 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 13:17:03,623 main DEBUG Took 0.009320 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 13:17:03,624 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 13:17:03,624 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 13:17:03,625 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 13:17:03,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,633 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 13:17:03,645 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,647 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,648 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,649 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,650 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,650 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,652 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,653 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,653 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,654 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,655 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,656 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,657 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 13:17:03,658 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,658 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 13:17:03,660 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 13:17:03,661 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 13:17:03,664 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 13:17:03,664 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 13:17:03,666 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 13:17:03,666 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 13:17:03,675 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 13:17:03,677 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 13:17:03,679 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 13:17:03,679 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 13:17:03,680 main DEBUG createAppenders(={Console}) 2024-12-02 13:17:03,680 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 13:17:03,681 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 13:17:03,681 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 13:17:03,682 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 13:17:03,682 main DEBUG OutputStream closed 2024-12-02 13:17:03,682 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 13:17:03,682 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 13:17:03,683 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 13:17:03,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 13:17:03,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 13:17:03,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 13:17:03,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 13:17:03,762 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 13:17:03,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 13:17:03,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 13:17:03,763 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 13:17:03,764 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 13:17:03,764 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 13:17:03,764 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 13:17:03,765 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 13:17:03,765 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 13:17:03,765 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 13:17:03,766 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 13:17:03,766 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 13:17:03,766 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 13:17:03,767 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 13:17:03,769 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 13:17:03,769 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 13:17:03,769 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 13:17:03,770 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T13:17:04,028 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076 2024-12-02 13:17:04,032 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 13:17:04,033 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T13:17:04,043 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T13:17:04,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T13:17:04,090 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6, deleteOnExit=true 2024-12-02T13:17:04,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T13:17:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/test.cache.data in system properties and HBase conf 2024-12-02T13:17:04,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T13:17:04,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.log.dir in system properties and HBase conf 2024-12-02T13:17:04,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T13:17:04,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T13:17:04,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T13:17:04,188 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T13:17:04,280 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T13:17:04,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T13:17:04,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T13:17:04,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T13:17:04,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T13:17:04,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T13:17:04,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T13:17:04,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T13:17:04,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T13:17:04,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T13:17:04,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/nfs.dump.dir in system properties and HBase conf 2024-12-02T13:17:04,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/java.io.tmpdir in system properties and HBase conf 2024-12-02T13:17:04,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T13:17:04,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T13:17:04,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T13:17:05,103 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T13:17:05,189 INFO [Time-limited test {}] log.Log(170): Logging initialized @2293ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T13:17:05,251 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T13:17:05,301 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T13:17:05,319 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T13:17:05,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T13:17:05,321 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T13:17:05,332 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T13:17:05,334 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.log.dir/,AVAILABLE} 2024-12-02T13:17:05,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T13:17:05,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/java.io.tmpdir/jetty-localhost-40833-hadoop-hdfs-3_4_1-tests_jar-_-any-4416936060093132125/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T13:17:05,537 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40833} 2024-12-02T13:17:05,538 INFO [Time-limited test {}] server.Server(415): Started @2642ms 2024-12-02T13:17:05,886 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T13:17:05,893 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T13:17:05,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T13:17:05,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T13:17:05,894 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T13:17:05,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.log.dir/,AVAILABLE} 2024-12-02T13:17:05,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T13:17:05,998 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/java.io.tmpdir/jetty-localhost-38223-hadoop-hdfs-3_4_1-tests_jar-_-any-6349235764618257028/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T13:17:05,999 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38223} 2024-12-02T13:17:05,999 INFO [Time-limited test {}] server.Server(415): Started @3103ms 2024-12-02T13:17:06,051 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T13:17:06,608 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data1/current/BP-1878892068-172.17.0.2-1733145424858/current, will proceed with Du for space computation calculation, 2024-12-02T13:17:06,608 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data2/current/BP-1878892068-172.17.0.2-1733145424858/current, will proceed with Du for space computation calculation, 2024-12-02T13:17:06,664 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T13:17:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ba89ea222ec00de with lease ID 0xb6a1dddf89ec9f21: Processing first storage report for DS-5a7d8704-85f5-4cd4-82c4-f0729657c738 from datanode DatanodeRegistration(127.0.0.1:37517, datanodeUuid=34595c3c-85ad-4c82-bb2d-22582016a5ec, infoPort=37751, infoSecurePort=0, ipcPort=40803, storageInfo=lv=-57;cid=testClusterID;nsid=1408998788;c=1733145424858) 2024-12-02T13:17:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ba89ea222ec00de with lease ID 0xb6a1dddf89ec9f21: from storage DS-5a7d8704-85f5-4cd4-82c4-f0729657c738 node DatanodeRegistration(127.0.0.1:37517, datanodeUuid=34595c3c-85ad-4c82-bb2d-22582016a5ec, infoPort=37751, infoSecurePort=0, ipcPort=40803, storageInfo=lv=-57;cid=testClusterID;nsid=1408998788;c=1733145424858), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-02T13:17:06,724 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ba89ea222ec00de with lease ID 0xb6a1dddf89ec9f21: Processing first storage report for DS-a91e596f-c7e9-4371-bd59-abca3a26f300 from datanode DatanodeRegistration(127.0.0.1:37517, datanodeUuid=34595c3c-85ad-4c82-bb2d-22582016a5ec, infoPort=37751, infoSecurePort=0, ipcPort=40803, storageInfo=lv=-57;cid=testClusterID;nsid=1408998788;c=1733145424858) 2024-12-02T13:17:06,725 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ba89ea222ec00de with lease ID 0xb6a1dddf89ec9f21: from storage DS-a91e596f-c7e9-4371-bd59-abca3a26f300 node DatanodeRegistration(127.0.0.1:37517, datanodeUuid=34595c3c-85ad-4c82-bb2d-22582016a5ec, infoPort=37751, infoSecurePort=0, ipcPort=40803, storageInfo=lv=-57;cid=testClusterID;nsid=1408998788;c=1733145424858), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T13:17:06,803 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076 2024-12-02T13:17:06,879 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/zookeeper_0, clientPort=60906, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T13:17:06,889 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60906 2024-12-02T13:17:06,902 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:06,906 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741825_1001 (size=7) 2024-12-02T13:17:07,557 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b with version=8 2024-12-02T13:17:07,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/hbase-staging 2024-12-02T13:17:07,633 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T13:17:07,831 INFO [Time-limited test {}] client.ConnectionUtils(128): master/06e3e18f2515:0 server-side Connection retries=45 2024-12-02T13:17:07,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:07,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:07,844 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T13:17:07,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:07,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T13:17:07,959 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T13:17:08,011 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T13:17:08,019 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T13:17:08,022 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T13:17:08,047 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94398 (auto-detected) 2024-12-02T13:17:08,048 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T13:17:08,072 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35237 2024-12-02T13:17:08,098 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35237 connecting to ZooKeeper ensemble=127.0.0.1:60906 2024-12-02T13:17:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352370x0, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T13:17:08,128 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35237-0x1009b13d81b0000 connected 2024-12-02T13:17:08,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:08,159 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:08,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T13:17:08,174 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b, hbase.cluster.distributed=false 2024-12-02T13:17:08,202 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T13:17:08,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35237 2024-12-02T13:17:08,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35237 2024-12-02T13:17:08,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35237 2024-12-02T13:17:08,209 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35237 2024-12-02T13:17:08,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35237 2024-12-02T13:17:08,314 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/06e3e18f2515:0 server-side Connection retries=45 2024-12-02T13:17:08,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:08,316 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:08,316 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T13:17:08,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T13:17:08,317 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T13:17:08,320 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T13:17:08,322 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T13:17:08,323 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45693 2024-12-02T13:17:08,325 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45693 connecting to ZooKeeper ensemble=127.0.0.1:60906 2024-12-02T13:17:08,327 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:08,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:08,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456930x0, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T13:17:08,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456930x0, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T13:17:08,338 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45693-0x1009b13d81b0001 connected 2024-12-02T13:17:08,340 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T13:17:08,347 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T13:17:08,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T13:17:08,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T13:17:08,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45693 2024-12-02T13:17:08,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45693 2024-12-02T13:17:08,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45693 2024-12-02T13:17:08,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45693 2024-12-02T13:17:08,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45693 2024-12-02T13:17:08,384 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;06e3e18f2515:35237 2024-12-02T13:17:08,385 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/06e3e18f2515,35237,1733145427680 2024-12-02T13:17:08,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T13:17:08,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T13:17:08,393 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/06e3e18f2515,35237,1733145427680 2024-12-02T13:17:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T13:17:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:08,418 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T13:17:08,419 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/06e3e18f2515,35237,1733145427680 from backup master directory 2024-12-02T13:17:08,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/06e3e18f2515,35237,1733145427680 2024-12-02T13:17:08,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T13:17:08,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T13:17:08,423 WARN [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T13:17:08,424 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=06e3e18f2515,35237,1733145427680 2024-12-02T13:17:08,426 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T13:17:08,429 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T13:17:08,482 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/hbase.id] with ID: 6274f614-26a8-4bfb-bcd2-91fd56d07722 2024-12-02T13:17:08,482 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/.tmp/hbase.id 2024-12-02T13:17:08,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741826_1002 (size=42) 2024-12-02T13:17:08,893 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/.tmp/hbase.id]:[hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/hbase.id] 2024-12-02T13:17:08,932 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T13:17:08,937 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T13:17:08,955 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-02T13:17:08,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:08,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:08,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741827_1003 (size=196) 2024-12-02T13:17:09,390 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:17:09,393 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T13:17:09,409 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:09,414 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T13:17:09,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741828_1004 (size=1189) 2024-12-02T13:17:09,857 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store 2024-12-02T13:17:09,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741829_1005 (size=34) 2024-12-02T13:17:10,277 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T13:17:10,280 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:10,281 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T13:17:10,281 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:17:10,281 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:17:10,282 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T13:17:10,283 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:17:10,283 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:17:10,284 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733145430281Disabling compacts and flushes for region at 1733145430281Disabling writes for close at 1733145430283 (+2 ms)Writing region close event to WAL at 1733145430283Closed at 1733145430283 2024-12-02T13:17:10,286 WARN [master/06e3e18f2515:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/.initializing 2024-12-02T13:17:10,286 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/WALs/06e3e18f2515,35237,1733145427680 2024-12-02T13:17:10,295 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T13:17:10,310 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=06e3e18f2515%2C35237%2C1733145427680, suffix=, logDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/WALs/06e3e18f2515,35237,1733145427680, archiveDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/oldWALs, maxLogs=10 2024-12-02T13:17:10,329 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/WALs/06e3e18f2515,35237,1733145427680/06e3e18f2515%2C35237%2C1733145427680.1733145430314, exclude list is [], retry=0 2024-12-02T13:17:10,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37517,DS-5a7d8704-85f5-4cd4-82c4-f0729657c738,DISK] 2024-12-02T13:17:10,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T13:17:10,373 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/WALs/06e3e18f2515,35237,1733145427680/06e3e18f2515%2C35237%2C1733145427680.1733145430314 2024-12-02T13:17:10,374 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751)] 2024-12-02T13:17:10,375 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:17:10,375 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:10,378 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,379 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T13:17:10,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:10,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:10,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T13:17:10,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:10,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:10,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T13:17:10,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:10,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:10,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T13:17:10,449 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:10,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:10,451 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,455 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,456 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,461 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,461 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,464 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T13:17:10,467 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T13:17:10,471 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:17:10,472 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74320557, jitterRate=0.10746259987354279}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T13:17:10,477 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733145430389Initializing all the Stores at 1733145430391 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145430391Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145430392 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145430392Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145430392Cleaning up temporary data from old regions at 1733145430461 (+69 ms)Region opened successfully at 1733145430477 (+16 ms) 2024-12-02T13:17:10,478 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T13:17:10,505 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d63692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=06e3e18f2515/172.17.0.2:0 2024-12-02T13:17:10,532 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T13:17:10,541 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T13:17:10,541 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T13:17:10,544 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T13:17:10,545 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T13:17:10,549 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T13:17:10,549 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T13:17:10,570 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T13:17:10,577 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T13:17:10,579 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T13:17:10,581 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T13:17:10,583 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T13:17:10,584 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T13:17:10,586 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T13:17:10,589 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T13:17:10,590 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T13:17:10,592 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T13:17:10,593 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T13:17:10,611 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T13:17:10,612 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T13:17:10,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T13:17:10,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T13:17:10,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,619 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=06e3e18f2515,35237,1733145427680, sessionid=0x1009b13d81b0000, setting cluster-up flag (Was=false) 2024-12-02T13:17:10,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,632 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T13:17:10,634 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=06e3e18f2515,35237,1733145427680 2024-12-02T13:17:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:10,642 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T13:17:10,644 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=06e3e18f2515,35237,1733145427680 2024-12-02T13:17:10,650 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T13:17:10,675 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(746): ClusterId : 6274f614-26a8-4bfb-bcd2-91fd56d07722 2024-12-02T13:17:10,677 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T13:17:10,681 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T13:17:10,681 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T13:17:10,683 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T13:17:10,684 DEBUG [RS:0;06e3e18f2515:45693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a05fcbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=06e3e18f2515/172.17.0.2:0 2024-12-02T13:17:10,696 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;06e3e18f2515:45693 2024-12-02T13:17:10,699 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T13:17:10,699 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T13:17:10,699 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T13:17:10,701 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(2659): reportForDuty to master=06e3e18f2515,35237,1733145427680 with port=45693, startcode=1733145428280 2024-12-02T13:17:10,713 DEBUG [RS:0;06e3e18f2515:45693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T13:17:10,715 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T13:17:10,724 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T13:17:10,732 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T13:17:10,738 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 06e3e18f2515,35237,1733145427680 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T13:17:10,747 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/06e3e18f2515:0, corePoolSize=5, maxPoolSize=5 2024-12-02T13:17:10,747 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/06e3e18f2515:0, corePoolSize=5, maxPoolSize=5 2024-12-02T13:17:10,747 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/06e3e18f2515:0, corePoolSize=5, maxPoolSize=5 2024-12-02T13:17:10,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T13:17:10,748 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/06e3e18f2515:0, corePoolSize=5, maxPoolSize=5 2024-12-02T13:17:10,748 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/06e3e18f2515:0, corePoolSize=10, maxPoolSize=10 2024-12-02T13:17:10,748 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,748 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/06e3e18f2515:0, corePoolSize=2, maxPoolSize=2 2024-12-02T13:17:10,749 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,750 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733145460750 2024-12-02T13:17:10,752 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T13:17:10,753 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T13:17:10,753 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35237 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:10,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T13:17:10,754 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T13:17:10,757 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T13:17:10,757 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T13:17:10,757 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T13:17:10,757 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T13:17:10,760 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:10,758 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,760 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T13:17:10,761 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T13:17:10,763 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T13:17:10,763 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T13:17:10,766 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T13:17:10,767 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T13:17:10,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741831_1007 (size=1321) 2024-12-02T13:17:10,773 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.large.0-1733145430768,5,FailOnTimeoutGroup] 2024-12-02T13:17:10,776 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.small.0-1733145430773,5,FailOnTimeoutGroup] 2024-12-02T13:17:10,776 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,777 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T13:17:10,778 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,778 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,781 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-02T13:17:10,781 WARN [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-02T13:17:10,883 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(2659): reportForDuty to master=06e3e18f2515,35237,1733145427680 with port=45693, startcode=1733145428280 2024-12-02T13:17:10,885 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35237 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 06e3e18f2515,45693,1733145428280 2024-12-02T13:17:10,887 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35237 {}] master.ServerManager(517): Registering regionserver=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:10,895 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:17:10,895 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44625 2024-12-02T13:17:10,895 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T13:17:10,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T13:17:10,901 DEBUG [RS:0;06e3e18f2515:45693 {}] zookeeper.ZKUtil(111): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/06e3e18f2515,45693,1733145428280 2024-12-02T13:17:10,901 WARN [RS:0;06e3e18f2515:45693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T13:17:10,901 INFO [RS:0;06e3e18f2515:45693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T13:17:10,901 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280 2024-12-02T13:17:10,901 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [06e3e18f2515,45693,1733145428280] 2024-12-02T13:17:10,927 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T13:17:10,942 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T13:17:10,947 INFO [RS:0;06e3e18f2515:45693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T13:17:10,948 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,950 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T13:17:10,956 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T13:17:10,958 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,959 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/06e3e18f2515:0, corePoolSize=2, maxPoolSize=2 2024-12-02T13:17:10,960 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,960 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,960 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,960 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,960 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,961 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/06e3e18f2515:0, corePoolSize=1, maxPoolSize=1 2024-12-02T13:17:10,961 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/06e3e18f2515:0, corePoolSize=3, maxPoolSize=3 2024-12-02T13:17:10,961 DEBUG [RS:0;06e3e18f2515:45693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0, corePoolSize=3, maxPoolSize=3 2024-12-02T13:17:10,963 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,964 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,964 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,964 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,964 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,964 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,45693,1733145428280-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T13:17:10,989 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T13:17:10,990 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,45693,1733145428280-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,990 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:10,991 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.Replication(171): 06e3e18f2515,45693,1733145428280 started 2024-12-02T13:17:11,007 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:11,007 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1482): Serving as 06e3e18f2515,45693,1733145428280, RpcServer on 06e3e18f2515/172.17.0.2:45693, sessionid=0x1009b13d81b0001 2024-12-02T13:17:11,008 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T13:17:11,008 DEBUG [RS:0;06e3e18f2515:45693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 06e3e18f2515,45693,1733145428280 2024-12-02T13:17:11,008 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '06e3e18f2515,45693,1733145428280' 2024-12-02T13:17:11,008 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T13:17:11,009 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T13:17:11,010 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T13:17:11,010 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T13:17:11,010 DEBUG [RS:0;06e3e18f2515:45693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 06e3e18f2515,45693,1733145428280 2024-12-02T13:17:11,010 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '06e3e18f2515,45693,1733145428280' 2024-12-02T13:17:11,010 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T13:17:11,011 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T13:17:11,011 DEBUG [RS:0;06e3e18f2515:45693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T13:17:11,011 INFO [RS:0;06e3e18f2515:45693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T13:17:11,011 INFO [RS:0;06e3e18f2515:45693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T13:17:11,116 INFO [RS:0;06e3e18f2515:45693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T13:17:11,119 INFO [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=06e3e18f2515%2C45693%2C1733145428280, suffix=, logDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280, archiveDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/oldWALs, maxLogs=32 2024-12-02T13:17:11,140 DEBUG [RS:0;06e3e18f2515:45693 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280/06e3e18f2515%2C45693%2C1733145428280.1733145431122, exclude list is [], retry=0 2024-12-02T13:17:11,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37517,DS-5a7d8704-85f5-4cd4-82c4-f0729657c738,DISK] 2024-12-02T13:17:11,148 INFO [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280/06e3e18f2515%2C45693%2C1733145428280.1733145431122 2024-12-02T13:17:11,149 DEBUG [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751)] 2024-12-02T13:17:11,174 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T13:17:11,175 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:17:11,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741833_1009 (size=32) 2024-12-02T13:17:11,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:11,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T13:17:11,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T13:17:11,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:11,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:11,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T13:17:11,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T13:17:11,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:11,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:11,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T13:17:11,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T13:17:11,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:11,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:11,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T13:17:11,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T13:17:11,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:11,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:11,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T13:17:11,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740 2024-12-02T13:17:11,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740 2024-12-02T13:17:11,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T13:17:11,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T13:17:11,606 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:17:11,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T13:17:11,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:17:11,612 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62711708, jitterRate=-0.06552273035049438}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:17:11,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733145431585Initializing all the Stores at 1733145431586 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145431586Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145431587 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145431587Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145431587Cleaning up temporary data from old regions at 1733145431606 (+19 ms)Region opened successfully at 1733145431615 (+9 ms) 2024-12-02T13:17:11,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T13:17:11,616 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T13:17:11,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T13:17:11,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T13:17:11,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T13:17:11,617 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T13:17:11,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733145431615Disabling compacts and flushes for region at 1733145431615Disabling writes for close at 1733145431616 (+1 ms)Writing region close event to WAL at 1733145431617 (+1 ms)Closed at 1733145431617 2024-12-02T13:17:11,621 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T13:17:11,621 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T13:17:11,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T13:17:11,633 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T13:17:11,636 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T13:17:11,788 DEBUG [06e3e18f2515:35237 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T13:17:11,799 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:11,805 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 06e3e18f2515,45693,1733145428280, state=OPENING 2024-12-02T13:17:11,810 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T13:17:11,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:11,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:17:11,812 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T13:17:11,812 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T13:17:11,814 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T13:17:11,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:11,988 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T13:17:11,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43539, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T13:17:12,001 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T13:17:12,002 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T13:17:12,003 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T13:17:12,006 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=06e3e18f2515%2C45693%2C1733145428280.meta, suffix=.meta, logDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280, archiveDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/oldWALs, maxLogs=32 2024-12-02T13:17:12,027 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280/06e3e18f2515%2C45693%2C1733145428280.meta.1733145432008.meta, exclude list is [], retry=0 2024-12-02T13:17:12,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37517,DS-5a7d8704-85f5-4cd4-82c4-f0729657c738,DISK] 2024-12-02T13:17:12,033 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/WALs/06e3e18f2515,45693,1733145428280/06e3e18f2515%2C45693%2C1733145428280.meta.1733145432008.meta 2024-12-02T13:17:12,034 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37751:37751)] 2024-12-02T13:17:12,034 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:17:12,036 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T13:17:12,038 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T13:17:12,042 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T13:17:12,046 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T13:17:12,046 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:12,046 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T13:17:12,046 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T13:17:12,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T13:17:12,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T13:17:12,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:12,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:12,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T13:17:12,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T13:17:12,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:12,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:12,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T13:17:12,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T13:17:12,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:12,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:12,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T13:17:12,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T13:17:12,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:12,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T13:17:12,060 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T13:17:12,062 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740 2024-12-02T13:17:12,065 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740 2024-12-02T13:17:12,068 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T13:17:12,068 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T13:17:12,069 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:17:12,072 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T13:17:12,073 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60276164, jitterRate=-0.10181516408920288}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:17:12,073 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T13:17:12,075 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733145432047Writing region info on filesystem at 1733145432047Initializing all the Stores at 1733145432049 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145432049Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145432049Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145432049Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733145432049Cleaning up temporary data from old regions at 1733145432068 (+19 ms)Running coprocessor post-open hooks at 1733145432074 (+6 ms)Region opened successfully at 1733145432075 (+1 ms) 2024-12-02T13:17:12,083 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733145431981 2024-12-02T13:17:12,093 DEBUG [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T13:17:12,093 INFO [RS_OPEN_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T13:17:12,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:12,097 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 06e3e18f2515,45693,1733145428280, state=OPEN 2024-12-02T13:17:12,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T13:17:12,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T13:17:12,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T13:17:12,100 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T13:17:12,101 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:12,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T13:17:12,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=06e3e18f2515,45693,1733145428280 in 285 msec 2024-12-02T13:17:12,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T13:17:12,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 482 msec 2024-12-02T13:17:12,115 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T13:17:12,115 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T13:17:12,138 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:12,139 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:12,165 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:12,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59353, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:12,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5130 sec 2024-12-02T13:17:12,195 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733145432195, completionTime=-1 2024-12-02T13:17:12,199 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T13:17:12,199 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T13:17:12,224 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T13:17:12,225 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733145492225 2024-12-02T13:17:12,225 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733145552225 2024-12-02T13:17:12,225 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-02T13:17:12,227 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,227 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,227 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,229 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-06e3e18f2515:35237, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,229 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,229 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,236 DEBUG [master/06e3e18f2515:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T13:17:12,254 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.830sec 2024-12-02T13:17:12,255 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T13:17:12,256 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T13:17:12,257 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T13:17:12,257 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T13:17:12,257 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T13:17:12,258 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T13:17:12,258 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T13:17:12,266 DEBUG [master/06e3e18f2515:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T13:17:12,266 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T13:17:12,267 INFO [master/06e3e18f2515:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=06e3e18f2515,35237,1733145427680-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T13:17:12,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:12,286 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T13:17:12,286 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T13:17:12,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:12,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:12,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aebee75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:12,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:12,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb1cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:12,319 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:12,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:12,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=06e3e18f2515,35237,1733145427680 2024-12-02T13:17:12,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=264, ProcessCount=11, AvailableMemoryMB=2587 2024-12-02T13:17:12,369 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:17:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ed11bb1 2024-12-02T13:17:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:17:12,407 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:17:12,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:17:12,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:17:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:12,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:17:12,435 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:12,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T13:17:12,438 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:17:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:12,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741835_1011 (size=1061) 2024-12-02T13:17:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:12,856 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:17:12,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741836_1012 (size=53) 2024-12-02T13:17:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f8f9c71492391b16a7ca84f91364ebb5, disabling compactions & flushes 2024-12-02T13:17:13,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. after waiting 0 ms 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,266 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,266 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8f9c71492391b16a7ca84f91364ebb5: Waiting for close lock at 1733145433266Disabling compacts and flushes for region at 1733145433266Disabling writes for close at 1733145433266Writing region close event to WAL at 1733145433266Closed at 1733145433266 2024-12-02T13:17:13,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:17:13,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145433269"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145433269"}]},"ts":"1733145433269"} 2024-12-02T13:17:13,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:17:13,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:17:13,281 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145433279"}]},"ts":"1733145433279"} 2024-12-02T13:17:13,285 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:17:13,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, ASSIGN}] 2024-12-02T13:17:13,288 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, ASSIGN 2024-12-02T13:17:13,290 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:17:13,442 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f8f9c71492391b16a7ca84f91364ebb5, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:13,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, ASSIGN because future has completed 2024-12-02T13:17:13,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:13,607 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,607 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:17:13,607 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,608 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:13,608 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,608 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,610 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,613 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:13,613 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f9c71492391b16a7ca84f91364ebb5 columnFamilyName A 2024-12-02T13:17:13,613 DEBUG [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:13,614 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(327): Store=f8f9c71492391b16a7ca84f91364ebb5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:13,614 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,616 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:13,617 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f9c71492391b16a7ca84f91364ebb5 columnFamilyName B 2024-12-02T13:17:13,617 DEBUG [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:13,617 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(327): Store=f8f9c71492391b16a7ca84f91364ebb5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:13,618 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,619 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:13,620 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f9c71492391b16a7ca84f91364ebb5 columnFamilyName C 2024-12-02T13:17:13,620 DEBUG [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:13,620 INFO [StoreOpener-f8f9c71492391b16a7ca84f91364ebb5-1 {}] regionserver.HStore(327): Store=f8f9c71492391b16a7ca84f91364ebb5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:13,621 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,621 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,623 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,624 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,626 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,626 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,627 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:17:13,631 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,635 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:17:13,637 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f8f9c71492391b16a7ca84f91364ebb5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63462190, jitterRate=-0.05433967709541321}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:17:13,637 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:13,638 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f8f9c71492391b16a7ca84f91364ebb5: Running coprocessor pre-open hook at 1733145433608Writing region info on filesystem at 1733145433608Initializing all the Stores at 1733145433609 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145433609Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145433610 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145433610Cleaning up temporary data from old regions at 1733145433626 (+16 ms)Running coprocessor post-open hooks at 1733145433637 (+11 ms)Region opened successfully at 1733145433638 (+1 ms) 2024-12-02T13:17:13,641 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., pid=6, masterSystemTime=1733145433601 2024-12-02T13:17:13,645 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,645 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:13,646 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f8f9c71492391b16a7ca84f91364ebb5, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:13,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:17:13,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T13:17:13,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 in 206 msec 2024-12-02T13:17:13,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T13:17:13,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, ASSIGN in 372 msec 2024-12-02T13:17:13,665 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:17:13,665 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145433665"}]},"ts":"1733145433665"} 2024-12-02T13:17:13,669 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:17:13,671 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:17:13,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-12-02T13:17:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T13:17:14,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:14,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775e68a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,602 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411c73ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,605 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6825ee03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,607 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,613 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:17:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:17:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b005fc9 2024-12-02T13:17:14,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:17:14,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:17:14,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,642 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,652 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38788dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35818969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100452bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,666 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65cace48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565e7015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec8a10e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af61fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db01178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,690 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7cbb29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f086cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:14,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:14,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c29b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:14,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:14,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78aaa9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:14,706 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,706 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,711 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:14,713 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,713 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,721 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T13:17:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,724 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:14,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:14,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:14,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:14,734 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,746 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,753 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,758 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,760 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,760 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,782 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:14,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:14,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:14,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:17:14,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:14,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:17:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:14,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:14,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T13:17:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:14,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:14,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:14,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145494928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145494926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145494924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:14,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145494939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145494957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:14,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b9089aebcbdb4dda8e3b0167682fa3f8 is 50, key is test_row_0/A:col10/1733145434799/Put/seqid=0 2024-12-02T13:17:15,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741837_1013 (size=12001) 2024-12-02T13:17:15,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b9089aebcbdb4dda8e3b0167682fa3f8 2024-12-02T13:17:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145495075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145495075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145495076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145495076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145495076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T13:17:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:15,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1fcdf4e0354c4ec5aada046b410495ac is 50, key is test_row_0/B:col10/1733145434799/Put/seqid=0 2024-12-02T13:17:15,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741838_1014 (size=12001) 2024-12-02T13:17:15,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1fcdf4e0354c4ec5aada046b410495ac 2024-12-02T13:17:15,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/101ed2a20fad4e199d27869d313d7e10 is 50, key is test_row_0/C:col10/1733145434799/Put/seqid=0 2024-12-02T13:17:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145495295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T13:17:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145495295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145495295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145495305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145495305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741839_1015 (size=12001) 2024-12-02T13:17:15,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/101ed2a20fad4e199d27869d313d7e10 2024-12-02T13:17:15,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b9089aebcbdb4dda8e3b0167682fa3f8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8 2024-12-02T13:17:15,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:17:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:15,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1fcdf4e0354c4ec5aada046b410495ac as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac 2024-12-02T13:17:15,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:17:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/101ed2a20fad4e199d27869d313d7e10 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10 2024-12-02T13:17:15,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:17:15,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f8f9c71492391b16a7ca84f91364ebb5 in 619ms, sequenceid=13, compaction requested=false 2024-12-02T13:17:15,431 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T13:17:15,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:15,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T13:17:15,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:15,464 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e58d33847d514b6c8dba7aa1ea2a477c is 50, key is test_row_0/A:col10/1733145434914/Put/seqid=0 2024-12-02T13:17:15,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741840_1016 (size=12001) 2024-12-02T13:17:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:15,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:15,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145495674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145495726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145495731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145495739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145495741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145495855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145495855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:15,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145495864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145495865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145495875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:15,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:15,934 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e58d33847d514b6c8dba7aa1ea2a477c 2024-12-02T13:17:15,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/3f35f3952227447fad7be326eaeaa310 is 50, key is test_row_0/B:col10/1733145434914/Put/seqid=0 2024-12-02T13:17:16,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741841_1017 (size=12001) 2024-12-02T13:17:16,014 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/3f35f3952227447fad7be326eaeaa310 2024-12-02T13:17:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/e71eb2ae3be34ef09c0f8e2fcb4a220c is 50, key is test_row_0/C:col10/1733145434914/Put/seqid=0 2024-12-02T13:17:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145496075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145496076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145496087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145496087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741842_1018 (size=12001) 2024-12-02T13:17:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145496095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145496384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145496385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145496395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145496405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145496405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,500 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/e71eb2ae3be34ef09c0f8e2fcb4a220c 2024-12-02T13:17:16,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e58d33847d514b6c8dba7aa1ea2a477c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c 2024-12-02T13:17:16,532 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T13:17:16,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/3f35f3952227447fad7be326eaeaa310 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310 2024-12-02T13:17:16,546 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T13:17:16,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/e71eb2ae3be34ef09c0f8e2fcb4a220c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c 2024-12-02T13:17:16,562 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-02T13:17:16,564 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f8f9c71492391b16a7ca84f91364ebb5 in 1100ms, sequenceid=33, compaction requested=false 2024-12-02T13:17:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T13:17:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T13:17:16,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T13:17:16,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-12-02T13:17:16,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8600 sec 2024-12-02T13:17:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T13:17:16,884 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:16,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:16,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T13:17:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:16,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:16,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:16,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:16,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:16,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/7ad3be706b9641e2a6e7df7682f261de is 50, key is test_row_0/A:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:16,920 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T13:17:16,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741843_1019 (size=12001) 2024-12-02T13:17:16,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/7ad3be706b9641e2a6e7df7682f261de 2024-12-02T13:17:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145496954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145496961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e7ac60684ea34e409d4b3f54a647dc26 is 50, key is test_row_0/B:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145496970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145496970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145496976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:16,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741844_1020 (size=12001) 2024-12-02T13:17:17,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e7ac60684ea34e409d4b3f54a647dc26 2024-12-02T13:17:17,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T13:17:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:17,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/aabfd35bfeee4e17a9f2cad2c15faadc is 50, key is test_row_0/C:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145497075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145497074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145497094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741845_1021 (size=12001) 2024-12-02T13:17:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/aabfd35bfeee4e17a9f2cad2c15faadc 2024-12-02T13:17:17,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145497104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145497105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/7ad3be706b9641e2a6e7df7682f261de as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de 2024-12-02T13:17:17,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:17:17,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e7ac60684ea34e409d4b3f54a647dc26 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26 2024-12-02T13:17:17,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:17:17,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/aabfd35bfeee4e17a9f2cad2c15faadc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc 2024-12-02T13:17:17,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:17:17,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for f8f9c71492391b16a7ca84f91364ebb5 in 284ms, sequenceid=47, compaction requested=true 2024-12-02T13:17:17,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:17,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:17,193 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:17,193 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:17,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:17,197 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:17,199 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:17,199 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,200 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=35.2 K 2024-12-02T13:17:17,204 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fcdf4e0354c4ec5aada046b410495ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145434723 2024-12-02T13:17:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:17,206 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f35f3952227447fad7be326eaeaa310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733145434914 2024-12-02T13:17:17,207 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ac60684ea34e409d4b3f54a647dc26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:17,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T13:17:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,210 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:17,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/0e5cd41995594d01ae001242aacd51cf is 50, key is test_row_0/A:col10/1733145436966/Put/seqid=0 2024-12-02T13:17:17,241 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:17,241 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:17,241 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,242 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=35.2 K 2024-12-02T13:17:17,242 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9089aebcbdb4dda8e3b0167682fa3f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145434723 2024-12-02T13:17:17,244 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58d33847d514b6c8dba7aa1ea2a477c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733145434914 2024-12-02T13:17:17,245 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad3be706b9641e2a6e7df7682f261de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:17,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741846_1022 (size=12001) 2024-12-02T13:17:17,291 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#10 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:17,297 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#11 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:17,298 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/df35e39d2cb54e5bae4214471b3e1080 is 50, key is test_row_0/A:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:17,299 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/2531723a944741ceb1471e3d43c9b3d3 is 50, key is test_row_0/B:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:17,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:17,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145497330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145497336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145497344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145497344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145497347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741847_1023 (size=12104) 2024-12-02T13:17:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741848_1024 (size=12104) 2024-12-02T13:17:17,403 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/df35e39d2cb54e5bae4214471b3e1080 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/df35e39d2cb54e5bae4214471b3e1080 2024-12-02T13:17:17,437 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into df35e39d2cb54e5bae4214471b3e1080(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:17,437 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:17,440 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145437190; duration=0sec 2024-12-02T13:17:17,441 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:17,441 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:17,442 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:17,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:17,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:17,444 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:17,445 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=35.2 K 2024-12-02T13:17:17,446 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101ed2a20fad4e199d27869d313d7e10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145434723 2024-12-02T13:17:17,447 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71eb2ae3be34ef09c0f8e2fcb4a220c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733145434914 2024-12-02T13:17:17,449 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting aabfd35bfeee4e17a9f2cad2c15faadc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145497455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145497465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145497484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,497 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:17,498 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/583a75431f0f48faa8d1f2cd76d89ffb is 50, key is test_row_0/C:col10/1733145435637/Put/seqid=0 2024-12-02T13:17:17,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145497495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145497495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:17,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741849_1025 (size=12104) 2024-12-02T13:17:17,554 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/583a75431f0f48faa8d1f2cd76d89ffb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/583a75431f0f48faa8d1f2cd76d89ffb 2024-12-02T13:17:17,556 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:17:17,570 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into 583a75431f0f48faa8d1f2cd76d89ffb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:17,570 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:17,570 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145437193; duration=0sec 2024-12-02T13:17:17,570 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:17,570 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:17,676 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/0e5cd41995594d01ae001242aacd51cf 2024-12-02T13:17:17,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145497674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145497684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e17806754b06402da30c21c3c6c49256 is 50, key is test_row_0/B:col10/1733145436966/Put/seqid=0 2024-12-02T13:17:17,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145497695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145497714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145497714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:17,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741850_1026 (size=12001) 2024-12-02T13:17:17,776 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/2531723a944741ceb1471e3d43c9b3d3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/2531723a944741ceb1471e3d43c9b3d3 2024-12-02T13:17:17,789 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 2531723a944741ceb1471e3d43c9b3d3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:17,789 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:17,789 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145437193; duration=0sec 2024-12-02T13:17:17,789 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:17,789 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145497985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T13:17:18,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T13:17:18,011 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T13:17:18,011 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T13:17:18,012 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T13:17:18,012 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T13:17:18,014 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T13:17:18,014 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T13:17:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145498004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145498004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:18,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145498025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145498026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e17806754b06402da30c21c3c6c49256 2024-12-02T13:17:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/a95218356ac44d6a9f6355001be1371b is 50, key is test_row_0/C:col10/1733145436966/Put/seqid=0 2024-12-02T13:17:18,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741851_1027 (size=12001) 2024-12-02T13:17:18,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145498494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145498524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145498525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145498534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145498544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:18,639 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/a95218356ac44d6a9f6355001be1371b 2024-12-02T13:17:18,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/0e5cd41995594d01ae001242aacd51cf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf 2024-12-02T13:17:18,664 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf, entries=150, sequenceid=66, filesize=11.7 K 2024-12-02T13:17:18,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T13:17:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e17806754b06402da30c21c3c6c49256 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256 2024-12-02T13:17:18,685 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256, entries=150, sequenceid=66, filesize=11.7 K 2024-12-02T13:17:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/a95218356ac44d6a9f6355001be1371b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b 2024-12-02T13:17:18,702 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b, entries=150, sequenceid=66, filesize=11.7 K 2024-12-02T13:17:18,704 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f8f9c71492391b16a7ca84f91364ebb5 in 1495ms, sequenceid=66, compaction requested=false 2024-12-02T13:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T13:17:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T13:17:18,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T13:17:18,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-02T13:17:18,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.8350 sec 2024-12-02T13:17:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T13:17:19,041 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:19,042 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T13:17:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T13:17:19,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:19,051 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:19,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T13:17:19,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T13:17:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:19,207 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:19,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:19,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/fefd2703eae84d4eadd1f3ec6ed0994e is 50, key is test_row_0/A:col10/1733145437334/Put/seqid=0 2024-12-02T13:17:19,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741852_1028 (size=12001) 2024-12-02T13:17:19,306 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/fefd2703eae84d4eadd1f3ec6ed0994e 2024-12-02T13:17:19,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/523a21bf93ec41f09b5f8233b11dc7c0 is 50, key is test_row_0/B:col10/1733145437334/Put/seqid=0 2024-12-02T13:17:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T13:17:19,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741853_1029 (size=12001) 2024-12-02T13:17:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:19,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145499563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145499566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145499567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145499569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145499569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T13:17:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145499685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145499686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145499686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145499686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145499686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,783 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/523a21bf93ec41f09b5f8233b11dc7c0 2024-12-02T13:17:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/eb8852c259b5482394cd0f75c0d88374 is 50, key is test_row_0/C:col10/1733145437334/Put/seqid=0 2024-12-02T13:17:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741854_1030 (size=12001) 2024-12-02T13:17:19,877 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/eb8852c259b5482394cd0f75c0d88374 2024-12-02T13:17:19,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/fefd2703eae84d4eadd1f3ec6ed0994e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e 2024-12-02T13:17:19,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145499895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145499894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145499896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145499896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145499895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:19,900 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T13:17:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/523a21bf93ec41f09b5f8233b11dc7c0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0 2024-12-02T13:17:19,912 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T13:17:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/eb8852c259b5482394cd0f75c0d88374 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374 2024-12-02T13:17:19,928 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T13:17:19,930 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f8f9c71492391b16a7ca84f91364ebb5 in 724ms, sequenceid=82, compaction requested=true 2024-12-02T13:17:19,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:19,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:19,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T13:17:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T13:17:19,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T13:17:19,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 882 msec 2024-12-02T13:17:19,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 897 msec 2024-12-02T13:17:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T13:17:20,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:20,186 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T13:17:20,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:20,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:20,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:20,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:20,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:17:20,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:20,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:20,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:20,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:20,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:20,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:20,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/ae76bb2cbeda43378cbcac1ced5e8e63 is 50, key is test_row_0/A:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145500251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741855_1031 (size=12001) 2024-12-02T13:17:20,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/ae76bb2cbeda43378cbcac1ced5e8e63 2024-12-02T13:17:20,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145500251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145500256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145500262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145500259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:20,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/a97da89d27094c43af56abf3dc1108d3 is 50, key is test_row_0/B:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:20,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741856_1032 (size=12001) 2024-12-02T13:17:20,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/a97da89d27094c43af56abf3dc1108d3 2024-12-02T13:17:20,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ca9e42d6471c4b4d8e0e28f822e135b7 is 50, key is test_row_0/C:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:20,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:20,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145500364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145500374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741857_1033 (size=12001) 2024-12-02T13:17:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145500384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145500384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145500394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:20,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:20,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145500574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145500584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145500595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145500595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145500604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:20,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:20,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ca9e42d6471c4b4d8e0e28f822e135b7 2024-12-02T13:17:20,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/ae76bb2cbeda43378cbcac1ced5e8e63 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63 2024-12-02T13:17:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:20,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:20,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T13:17:20,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/a97da89d27094c43af56abf3dc1108d3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3 2024-12-02T13:17:20,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T13:17:20,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ca9e42d6471c4b4d8e0e28f822e135b7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7 2024-12-02T13:17:20,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T13:17:20,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for f8f9c71492391b16a7ca84f91364ebb5 in 650ms, sequenceid=104, compaction requested=true 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:20,864 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:20,864 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:20,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:20,867 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:20,867 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:20,867 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:20,867 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:20,867 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,868 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,868 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/df35e39d2cb54e5bae4214471b3e1080, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=47.0 K 2024-12-02T13:17:20,868 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/2531723a944741ceb1471e3d43c9b3d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=47.0 K 2024-12-02T13:17:20,868 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting df35e39d2cb54e5bae4214471b3e1080, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:20,868 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2531723a944741ceb1471e3d43c9b3d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:20,869 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e17806754b06402da30c21c3c6c49256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733145436936 2024-12-02T13:17:20,869 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5cd41995594d01ae001242aacd51cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733145436936 2024-12-02T13:17:20,870 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 523a21bf93ec41f09b5f8233b11dc7c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145437334 2024-12-02T13:17:20,870 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fefd2703eae84d4eadd1f3ec6ed0994e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145437334 2024-12-02T13:17:20,870 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a97da89d27094c43af56abf3dc1108d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:20,870 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae76bb2cbeda43378cbcac1ced5e8e63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:20,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:20,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:17:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:20,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:20,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:20,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:20,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:20,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:20,896 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:20,897 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/024b873875c34e6cb14931d3b4bcf80b is 50, key is test_row_0/B:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:20,908 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:20,908 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/a095d09c523944a59e4e3975d4ae2843 is 50, key is test_row_0/A:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:20,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/40763a6de7694d18b02c1126519dda90 is 50, key is test_row_0/A:col10/1733145440250/Put/seqid=0 2024-12-02T13:17:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741858_1034 (size=12241) 2024-12-02T13:17:20,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:20,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:20,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145440945, max timestamp=1733145440965Num uniques 859; 2024-12-02T13:17:20,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:20,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741859_1035 (size=12241) 2024-12-02T13:17:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145440945, max timestamp=1733145440965Num uniques 859; 2024-12-02T13:17:21,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:21,010 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/024b873875c34e6cb14931d3b4bcf80b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/024b873875c34e6cb14931d3b4bcf80b 2024-12-02T13:17:21,011 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/a095d09c523944a59e4e3975d4ae2843 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/a095d09c523944a59e4e3975d4ae2843 2024-12-02T13:17:21,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145440945, max timestamp=1733145440967Num uniques 859; 2024-12-02T13:17:21,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:21,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741860_1036 (size=14341) 2024-12-02T13:17:21,061 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 024b873875c34e6cb14931d3b4bcf80b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:21,061 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into a095d09c523944a59e4e3975d4ae2843(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:21,061 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:21,061 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=12, startTime=1733145440864; duration=0sec 2024-12-02T13:17:21,061 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:21,061 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:21,062 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:21,062 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:21,063 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=12, startTime=1733145440864; duration=0sec 2024-12-02T13:17:21,063 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:21,063 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:21,067 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:21,068 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:21,068 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,068 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/583a75431f0f48faa8d1f2cd76d89ffb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=47.0 K 2024-12-02T13:17:21,069 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 583a75431f0f48faa8d1f2cd76d89ffb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145435637 2024-12-02T13:17:21,070 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a95218356ac44d6a9f6355001be1371b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733145436936 2024-12-02T13:17:21,070 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eb8852c259b5482394cd0f75c0d88374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145437334 2024-12-02T13:17:21,071 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9e42d6471c4b4d8e0e28f822e135b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:21,104 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:21,104 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/4a4da4e4bba040cb98b1251203e5eded is 50, key is test_row_0/C:col10/1733145439560/Put/seqid=0 2024-12-02T13:17:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145501102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145501108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145501118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145501126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145501126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741861_1037 (size=12241) 2024-12-02T13:17:21,171 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/4a4da4e4bba040cb98b1251203e5eded as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4a4da4e4bba040cb98b1251203e5eded 2024-12-02T13:17:21,185 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into 4a4da4e4bba040cb98b1251203e5eded(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:21,185 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:21,185 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=12, startTime=1733145440864; duration=0sec 2024-12-02T13:17:21,185 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:21,186 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145501236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145501236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145501244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145501244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145501257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:21,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/40763a6de7694d18b02c1126519dda90 2024-12-02T13:17:21,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145501444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145501444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145501465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145501466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e6d55d57f96f4e71843c11541f3e1320 is 50, key is test_row_0/B:col10/1733145440250/Put/seqid=0 2024-12-02T13:17:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145501477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741862_1038 (size=12001) 2024-12-02T13:17:21,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e6d55d57f96f4e71843c11541f3e1320 2024-12-02T13:17:21,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1d54bd69f73b44f181dc89675334654d is 50, key is test_row_0/C:col10/1733145440250/Put/seqid=0 2024-12-02T13:17:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741863_1039 (size=12001) 2024-12-02T13:17:21,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145501764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145501764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145501774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145501774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145501794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:21,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:21,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:22,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1d54bd69f73b44f181dc89675334654d 2024-12-02T13:17:22,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/40763a6de7694d18b02c1126519dda90 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90 2024-12-02T13:17:22,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90, entries=200, sequenceid=115, filesize=14.0 K 2024-12-02T13:17:22,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/e6d55d57f96f4e71843c11541f3e1320 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320 2024-12-02T13:17:22,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320, entries=150, sequenceid=115, filesize=11.7 K 2024-12-02T13:17:22,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1d54bd69f73b44f181dc89675334654d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d 2024-12-02T13:17:22,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:22,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:22,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:22,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:22,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d, entries=150, sequenceid=115, filesize=11.7 K 2024-12-02T13:17:22,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f8f9c71492391b16a7ca84f91364ebb5 in 1181ms, sequenceid=115, compaction requested=false 2024-12-02T13:17:22,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:22,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T13:17:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:22,222 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T13:17:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:22,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/bbd3ea66277c4ac39a7bd8c5446eec1e is 50, key is test_row_0/A:col10/1733145441090/Put/seqid=0 2024-12-02T13:17:22,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:22,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741864_1040 (size=12151) 2024-12-02T13:17:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145502294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145502301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145502308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145502309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145502315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145502424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145502424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145502424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145502634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145502634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145502635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:22,693 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/bbd3ea66277c4ac39a7bd8c5446eec1e 2024-12-02T13:17:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cea02bd8588742b095e40cbed4eab7da is 50, key is test_row_0/B:col10/1733145441090/Put/seqid=0 2024-12-02T13:17:22,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741865_1041 (size=12151) 2024-12-02T13:17:22,791 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cea02bd8588742b095e40cbed4eab7da 2024-12-02T13:17:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2c669de185a1487ba6adcaa0deea3c65 is 50, key is test_row_0/C:col10/1733145441090/Put/seqid=0 2024-12-02T13:17:22,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741866_1042 (size=12151) 2024-12-02T13:17:22,880 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2c669de185a1487ba6adcaa0deea3c65 2024-12-02T13:17:22,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/bbd3ea66277c4ac39a7bd8c5446eec1e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e 2024-12-02T13:17:22,899 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e, entries=150, sequenceid=154, filesize=11.9 K 2024-12-02T13:17:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cea02bd8588742b095e40cbed4eab7da as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da 2024-12-02T13:17:22,910 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da, entries=150, sequenceid=154, filesize=11.9 K 2024-12-02T13:17:22,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2c669de185a1487ba6adcaa0deea3c65 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65 2024-12-02T13:17:22,924 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65, entries=150, sequenceid=154, filesize=11.9 K 2024-12-02T13:17:22,927 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for f8f9c71492391b16a7ca84f91364ebb5 in 705ms, sequenceid=154, compaction requested=true 2024-12-02T13:17:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:22,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T13:17:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T13:17:22,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T13:17:22,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7350 sec 2024-12-02T13:17:22,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.7530 sec 2024-12-02T13:17:22,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:22,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:22,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/272df6d3397c4a4b9cc75ab98f40ed4d is 50, key is test_row_0/A:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741867_1043 (size=16931) 2024-12-02T13:17:23,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/272df6d3397c4a4b9cc75ab98f40ed4d 2024-12-02T13:17:23,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c211694e357d43338b56eb36467fb49d is 50, key is test_row_0/B:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145503017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145503014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145503024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741868_1044 (size=12151) 2024-12-02T13:17:23,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c211694e357d43338b56eb36467fb49d 2024-12-02T13:17:23,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf is 50, key is test_row_0/C:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741869_1045 (size=12151) 2024-12-02T13:17:23,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145503133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145503133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145503134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145503324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,329 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145503344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145503345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145503345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145503345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,351 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf 2024-12-02T13:17:23,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/272df6d3397c4a4b9cc75ab98f40ed4d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d 2024-12-02T13:17:23,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d, entries=250, sequenceid=167, filesize=16.5 K 2024-12-02T13:17:23,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c211694e357d43338b56eb36467fb49d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d 2024-12-02T13:17:23,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d, entries=150, sequenceid=167, filesize=11.9 K 2024-12-02T13:17:23,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf 2024-12-02T13:17:23,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf, entries=150, sequenceid=167, filesize=11.9 K 2024-12-02T13:17:23,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f8f9c71492391b16a7ca84f91364ebb5 in 618ms, sequenceid=167, compaction requested=true 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:23,567 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:23,567 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:23,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:23,569 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:23,569 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:23,569 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:23,569 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:23,569 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:23,569 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:23,569 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/024b873875c34e6cb14931d3b4bcf80b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=47.4 K 2024-12-02T13:17:23,569 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/a095d09c523944a59e4e3975d4ae2843, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=54.4 K 2024-12-02T13:17:23,571 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a095d09c523944a59e4e3975d4ae2843, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:23,571 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 024b873875c34e6cb14931d3b4bcf80b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:23,572 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d55d57f96f4e71843c11541f3e1320, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733145440250 2024-12-02T13:17:23,572 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40763a6de7694d18b02c1126519dda90, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733145440250 2024-12-02T13:17:23,573 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cea02bd8588742b095e40cbed4eab7da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733145441090 2024-12-02T13:17:23,573 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbd3ea66277c4ac39a7bd8c5446eec1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733145441090 2024-12-02T13:17:23,573 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c211694e357d43338b56eb36467fb49d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442302 2024-12-02T13:17:23,574 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272df6d3397c4a4b9cc75ab98f40ed4d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442291 2024-12-02T13:17:23,599 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:23,600 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/0d217415bf224dd08e06ded6388ba944 is 50, key is test_row_0/B:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,611 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:23,612 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/2049060d94544261b29c4155e07bb59e is 50, key is test_row_0/A:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:23,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:17:23,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741870_1046 (size=12527) 2024-12-02T13:17:23,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741871_1047 (size=12527) 2024-12-02T13:17:23,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/2049060d94544261b29c4155e07bb59e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2049060d94544261b29c4155e07bb59e 2024-12-02T13:17:23,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145443011, max timestamp=1733145443659Num uniques 859; 2024-12-02T13:17:23,687 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:23,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:23,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:23,694 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into 2049060d94544261b29c4155e07bb59e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:23,694 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:23,694 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=12, startTime=1733145443567; duration=0sec 2024-12-02T13:17:23,694 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:23,694 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:23,694 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:23,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:23,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:23,704 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:23,704 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4a4da4e4bba040cb98b1251203e5eded, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=47.4 K 2024-12-02T13:17:23,705 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4da4e4bba040cb98b1251203e5eded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733145439560 2024-12-02T13:17:23,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/2268b7ecc10b472b83bb0b9b1905d49d is 50, key is test_row_0/A:col10/1733145443011/Put/seqid=0 2024-12-02T13:17:23,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d54bd69f73b44f181dc89675334654d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733145440250 2024-12-02T13:17:23,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c669de185a1487ba6adcaa0deea3c65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733145441090 2024-12-02T13:17:23,707 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9dde7b4fcd54f05a0f5b65d4aa42eaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442302 2024-12-02T13:17:23,739 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:23,740 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/14b009bf42d2496cb89545e17980d848 is 50, key is test_row_0/C:col10/1733145442946/Put/seqid=0 2024-12-02T13:17:23,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145503735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145503740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145503743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741872_1048 (size=14541) 2024-12-02T13:17:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741873_1049 (size=12527) 2024-12-02T13:17:23,836 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/14b009bf42d2496cb89545e17980d848 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/14b009bf42d2496cb89545e17980d848 2024-12-02T13:17:23,845 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into 14b009bf42d2496cb89545e17980d848(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:23,845 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:23,846 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=12, startTime=1733145443567; duration=0sec 2024-12-02T13:17:23,846 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:23,846 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145503854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145503855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:23,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145503854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145504064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145504074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145504075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,082 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/0d217415bf224dd08e06ded6388ba944 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0d217415bf224dd08e06ded6388ba944 2024-12-02T13:17:24,104 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 0d217415bf224dd08e06ded6388ba944(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:24,105 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:24,105 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=12, startTime=1733145443567; duration=0sec 2024-12-02T13:17:24,105 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:24,105 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:24,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/2268b7ecc10b472b83bb0b9b1905d49d 2024-12-02T13:17:24,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c12f015bbbee4818bffe95829b959f56 is 50, key is test_row_0/B:col10/1733145443011/Put/seqid=0 2024-12-02T13:17:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741874_1050 (size=12151) 2024-12-02T13:17:24,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c12f015bbbee4818bffe95829b959f56 2024-12-02T13:17:24,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c405070c2e349aaa436bc7d727ba6ba is 50, key is test_row_0/C:col10/1733145443011/Put/seqid=0 2024-12-02T13:17:24,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741875_1051 (size=12151) 2024-12-02T13:17:24,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c405070c2e349aaa436bc7d727ba6ba 2024-12-02T13:17:24,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/2268b7ecc10b472b83bb0b9b1905d49d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d 2024-12-02T13:17:24,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d, entries=200, sequenceid=189, filesize=14.2 K 2024-12-02T13:17:24,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c12f015bbbee4818bffe95829b959f56 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56 2024-12-02T13:17:24,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56, entries=150, sequenceid=189, filesize=11.9 K 2024-12-02T13:17:24,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c405070c2e349aaa436bc7d727ba6ba as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba 2024-12-02T13:17:24,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba, entries=150, sequenceid=189, filesize=11.9 K 2024-12-02T13:17:24,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for f8f9c71492391b16a7ca84f91364ebb5 in 634ms, sequenceid=189, compaction requested=false 2024-12-02T13:17:24,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T13:17:24,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:24,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T13:17:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:24,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:24,351 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:24,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:24,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:24,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:24,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d971660d77d145d78d6ee2482a9d4fd7 is 50, key is test_row_0/A:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:24,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741876_1052 (size=14541) 2024-12-02T13:17:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145504439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145504439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145504442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:24,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145504555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145504560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145504560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:24,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145504763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145504774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145504774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:24,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d971660d77d145d78d6ee2482a9d4fd7 2024-12-02T13:17:24,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/7fbdaf47c3664e6986fd4397eb490b88 is 50, key is test_row_0/B:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741877_1053 (size=12151) 2024-12-02T13:17:24,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/7fbdaf47c3664e6986fd4397eb490b88 2024-12-02T13:17:24,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/04afeaa7d6164991be205d0f67161c0d is 50, key is test_row_0/C:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:24,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:24,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:24,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741878_1054 (size=12151) 2024-12-02T13:17:25,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145505074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145505094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145505095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145505344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,356 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145505374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,386 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/04afeaa7d6164991be205d0f67161c0d 2024-12-02T13:17:25,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d971660d77d145d78d6ee2482a9d4fd7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7 2024-12-02T13:17:25,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7, entries=200, sequenceid=208, filesize=14.2 K 2024-12-02T13:17:25,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/7fbdaf47c3664e6986fd4397eb490b88 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88 2024-12-02T13:17:25,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T13:17:25,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/04afeaa7d6164991be205d0f67161c0d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d 2024-12-02T13:17:25,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T13:17:25,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f8f9c71492391b16a7ca84f91364ebb5 in 1074ms, sequenceid=208, compaction requested=true 2024-12-02T13:17:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:25,454 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:25,455 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:25,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:25,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:25,457 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:25,457 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:25,457 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,458 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0d217415bf224dd08e06ded6388ba944, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.0 K 2024-12-02T13:17:25,458 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:25,458 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:25,459 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,459 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2049060d94544261b29c4155e07bb59e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=40.6 K 2024-12-02T13:17:25,459 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2049060d94544261b29c4155e07bb59e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442302 2024-12-02T13:17:25,460 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d217415bf224dd08e06ded6388ba944, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442302 2024-12-02T13:17:25,461 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2268b7ecc10b472b83bb0b9b1905d49d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733145443011 2024-12-02T13:17:25,461 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c12f015bbbee4818bffe95829b959f56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733145443011 2024-12-02T13:17:25,463 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d971660d77d145d78d6ee2482a9d4fd7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443719 2024-12-02T13:17:25,463 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbdaf47c3664e6986fd4397eb490b88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443737 2024-12-02T13:17:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:25,487 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:25,488 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c80074203b164cf4844bee345b79a35c is 50, key is test_row_0/B:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:25,502 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#43 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:25,503 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/61a84814840b48da834e1b4502f9268e is 50, key is test_row_0/A:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741880_1056 (size=12629) 2024-12-02T13:17:25,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741879_1055 (size=12629) 2024-12-02T13:17:25,555 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/61a84814840b48da834e1b4502f9268e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/61a84814840b48da834e1b4502f9268e 2024-12-02T13:17:25,565 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c80074203b164cf4844bee345b79a35c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c80074203b164cf4844bee345b79a35c 2024-12-02T13:17:25,573 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into c80074203b164cf4844bee345b79a35c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:25,573 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into 61a84814840b48da834e1b4502f9268e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:25,573 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145445454; duration=0sec 2024-12-02T13:17:25,573 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145445454; duration=0sec 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:25,573 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:25,574 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:25,575 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:25,575 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:25,575 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,575 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/14b009bf42d2496cb89545e17980d848, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.0 K 2024-12-02T13:17:25,578 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b009bf42d2496cb89545e17980d848, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733145442302 2024-12-02T13:17:25,578 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c405070c2e349aaa436bc7d727ba6ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733145443011 2024-12-02T13:17:25,579 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 04afeaa7d6164991be205d0f67161c0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443737 2024-12-02T13:17:25,591 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:25,592 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/89dfc70e11234ea88c15f5144832f152 is 50, key is test_row_0/C:col10/1733145443737/Put/seqid=0 2024-12-02T13:17:25,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T13:17:25,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/953e5d21e3ca43948e49654f08d500b0 is 50, key is test_row_0/A:col10/1733145445593/Put/seqid=0 2024-12-02T13:17:25,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741881_1057 (size=12629) 2024-12-02T13:17:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145505632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145505634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/89dfc70e11234ea88c15f5144832f152 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/89dfc70e11234ea88c15f5144832f152 2024-12-02T13:17:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145505638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,656 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into 89dfc70e11234ea88c15f5144832f152(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:25,656 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:25,656 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145445455; duration=0sec 2024-12-02T13:17:25,657 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:25,657 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741882_1058 (size=14541) 2024-12-02T13:17:25,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/953e5d21e3ca43948e49654f08d500b0 2024-12-02T13:17:25,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f07092a356584c7eb4c84441d511dc62 is 50, key is test_row_0/B:col10/1733145445593/Put/seqid=0 2024-12-02T13:17:25,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741883_1059 (size=12151) 2024-12-02T13:17:25,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f07092a356584c7eb4c84441d511dc62 2024-12-02T13:17:25,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145505744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145505751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145505754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:25,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:25,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2dbd98a4bc664ecaa4703de7b1c18f6d is 50, key is test_row_0/C:col10/1733145445593/Put/seqid=0 2024-12-02T13:17:25,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741884_1060 (size=12151) 2024-12-02T13:17:25,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2dbd98a4bc664ecaa4703de7b1c18f6d 2024-12-02T13:17:25,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/953e5d21e3ca43948e49654f08d500b0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0 2024-12-02T13:17:25,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0, entries=200, sequenceid=228, filesize=14.2 K 2024-12-02T13:17:25,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f07092a356584c7eb4c84441d511dc62 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62 2024-12-02T13:17:25,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T13:17:25,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/2dbd98a4bc664ecaa4703de7b1c18f6d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d 2024-12-02T13:17:25,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d, entries=150, sequenceid=228, filesize=11.9 K 2024-12-02T13:17:25,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f8f9c71492391b16a7ca84f91364ebb5 in 274ms, sequenceid=228, compaction requested=false 2024-12-02T13:17:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:25,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:25,910 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/59ec16ec9af24c13b76f7e4d3a212409 is 50, key is test_row_0/A:col10/1733145445635/Put/seqid=0 2024-12-02T13:17:25,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741885_1061 (size=12151) 2024-12-02T13:17:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:25,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145506036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145506035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145506043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145506154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145506154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145506157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145506364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145506365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145506365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,372 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/59ec16ec9af24c13b76f7e4d3a212409 2024-12-02T13:17:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/eeec56ca8c6a4764ab88b87d83821e51 is 50, key is test_row_0/B:col10/1733145445635/Put/seqid=0 2024-12-02T13:17:26,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741886_1062 (size=12151) 2024-12-02T13:17:26,409 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/eeec56ca8c6a4764ab88b87d83821e51 2024-12-02T13:17:26,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ce63b4789f724f71975215451d2bdfa4 is 50, key is test_row_0/C:col10/1733145445635/Put/seqid=0 2024-12-02T13:17:26,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741887_1063 (size=12151) 2024-12-02T13:17:26,478 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ce63b4789f724f71975215451d2bdfa4 2024-12-02T13:17:26,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/59ec16ec9af24c13b76f7e4d3a212409 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409 2024-12-02T13:17:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:26,501 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409, entries=150, sequenceid=243, filesize=11.9 K 2024-12-02T13:17:26,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/eeec56ca8c6a4764ab88b87d83821e51 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51 2024-12-02T13:17:26,513 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51, entries=150, sequenceid=243, filesize=11.9 K 2024-12-02T13:17:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ce63b4789f724f71975215451d2bdfa4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4 2024-12-02T13:17:26,536 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4, entries=150, sequenceid=243, filesize=11.9 K 2024-12-02T13:17:26,539 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f8f9c71492391b16a7ca84f91364ebb5 in 629ms, sequenceid=243, compaction requested=true 2024-12-02T13:17:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T13:17:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T13:17:26,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T13:17:26,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1900 sec 2024-12-02T13:17:26,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2030 sec 2024-12-02T13:17:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:26,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:26,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:26,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/55a80b880a5f4bdb8d895fd8f862464b is 50, key is test_row_0/A:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741888_1064 (size=12301) 2024-12-02T13:17:26,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145506695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145506698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145506698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145506803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145506814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145506815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:26,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145507014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145507024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145507024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/55a80b880a5f4bdb8d895fd8f862464b 2024-12-02T13:17:27,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f04ae371a52948b58d256a672e03f6e2 is 50, key is test_row_0/B:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:27,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741889_1065 (size=12301) 2024-12-02T13:17:27,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f04ae371a52948b58d256a672e03f6e2 2024-12-02T13:17:27,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bee3b8e149a447a94e84e9478fa681c is 50, key is test_row_0/C:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741890_1066 (size=12301) 2024-12-02T13:17:27,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145507324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145507334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145507334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bee3b8e149a447a94e84e9478fa681c 2024-12-02T13:17:27,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/55a80b880a5f4bdb8d895fd8f862464b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b 2024-12-02T13:17:27,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b, entries=150, sequenceid=262, filesize=12.0 K 2024-12-02T13:17:27,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/f04ae371a52948b58d256a672e03f6e2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2 2024-12-02T13:17:27,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2, entries=150, sequenceid=262, filesize=12.0 K 2024-12-02T13:17:27,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bee3b8e149a447a94e84e9478fa681c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c 2024-12-02T13:17:27,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c, entries=150, sequenceid=262, filesize=12.0 K 2024-12-02T13:17:27,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f8f9c71492391b16a7ca84f91364ebb5 in 968ms, sequenceid=262, compaction requested=true 2024-12-02T13:17:27,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:27,645 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:27,645 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:27,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:27,646 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51622 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:27,647 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:27,647 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:27,647 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/61a84814840b48da834e1b4502f9268e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=50.4 K 2024-12-02T13:17:27,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:27,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:27,647 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:27,647 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c80074203b164cf4844bee345b79a35c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=48.1 K 2024-12-02T13:17:27,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a84814840b48da834e1b4502f9268e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443737 2024-12-02T13:17:27,648 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c80074203b164cf4844bee345b79a35c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443737 2024-12-02T13:17:27,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953e5d21e3ca43948e49654f08d500b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733145444408 2024-12-02T13:17:27,648 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f07092a356584c7eb4c84441d511dc62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733145444408 2024-12-02T13:17:27,649 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59ec16ec9af24c13b76f7e4d3a212409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733145445628 2024-12-02T13:17:27,649 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55a80b880a5f4bdb8d895fd8f862464b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:27,649 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eeec56ca8c6a4764ab88b87d83821e51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733145445628 2024-12-02T13:17:27,650 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f04ae371a52948b58d256a672e03f6e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:27,672 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:27,672 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/124112ffb9c441c1a90edeac045a4036 is 50, key is test_row_0/A:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:27,685 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:27,685 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/a5da019a8b664a1ca91203f725e6c9d8 is 50, key is test_row_0/B:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741891_1067 (size=12915) 2024-12-02T13:17:27,745 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/124112ffb9c441c1a90edeac045a4036 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/124112ffb9c441c1a90edeac045a4036 2024-12-02T13:17:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741892_1068 (size=12915) 2024-12-02T13:17:27,758 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into 124112ffb9c441c1a90edeac045a4036(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:27,758 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:27,758 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=12, startTime=1733145447644; duration=0sec 2024-12-02T13:17:27,758 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:27,758 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:27,758 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:17:27,763 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/a5da019a8b664a1ca91203f725e6c9d8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a5da019a8b664a1ca91203f725e6c9d8 2024-12-02T13:17:27,770 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:17:27,770 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:27,770 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:27,786 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/89dfc70e11234ea88c15f5144832f152, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=48.1 K 2024-12-02T13:17:27,787 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89dfc70e11234ea88c15f5144832f152, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733145443737 2024-12-02T13:17:27,788 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dbd98a4bc664ecaa4703de7b1c18f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733145444408 2024-12-02T13:17:27,788 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce63b4789f724f71975215451d2bdfa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733145445628 2024-12-02T13:17:27,789 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bee3b8e149a447a94e84e9478fa681c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:27,790 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into a5da019a8b664a1ca91203f725e6c9d8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:27,790 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:27,790 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=12, startTime=1733145447645; duration=0sec 2024-12-02T13:17:27,791 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:27,791 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:27,817 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#56 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:27,818 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/c1c341caaaa0467ca6a66bc3a685e16f is 50, key is test_row_0/C:col10/1733145446674/Put/seqid=0 2024-12-02T13:17:27,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:27,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:27,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:27,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741893_1069 (size=12915) 2024-12-02T13:17:27,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e63745357cce4d51adb9827e30624d30 is 50, key is test_row_0/A:col10/1733145447846/Put/seqid=0 2024-12-02T13:17:27,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145507891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145507895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145507898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741894_1070 (size=17181) 2024-12-02T13:17:27,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e63745357cce4d51adb9827e30624d30 2024-12-02T13:17:27,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/b8851ed8282149cd842daec55ba8226c is 50, key is test_row_0/B:col10/1733145447846/Put/seqid=0 2024-12-02T13:17:27,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741895_1071 (size=12301) 2024-12-02T13:17:27,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/b8851ed8282149cd842daec55ba8226c 2024-12-02T13:17:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145508005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145508005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145508014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1480810a813748ca89b283da2fd2ed8e is 50, key is test_row_0/C:col10/1733145447846/Put/seqid=0 2024-12-02T13:17:28,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741896_1072 (size=12301) 2024-12-02T13:17:28,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1480810a813748ca89b283da2fd2ed8e 2024-12-02T13:17:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/e63745357cce4d51adb9827e30624d30 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30 2024-12-02T13:17:28,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30, entries=250, sequenceid=279, filesize=16.8 K 2024-12-02T13:17:28,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/b8851ed8282149cd842daec55ba8226c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c 2024-12-02T13:17:28,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c, entries=150, sequenceid=279, filesize=12.0 K 2024-12-02T13:17:28,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1480810a813748ca89b283da2fd2ed8e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e 2024-12-02T13:17:28,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e, entries=150, sequenceid=279, filesize=12.0 K 2024-12-02T13:17:28,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f8f9c71492391b16a7ca84f91364ebb5 in 227ms, sequenceid=279, compaction requested=false 2024-12-02T13:17:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:28,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:17:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:28,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/9bd63c34df71420990656eca73f8c6e8 is 50, key is test_row_0/A:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741897_1073 (size=14741) 2024-12-02T13:17:28,273 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/c1c341caaaa0467ca6a66bc3a685e16f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c1c341caaaa0467ca6a66bc3a685e16f 2024-12-02T13:17:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145508260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145508263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145508266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,283 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into c1c341caaaa0467ca6a66bc3a685e16f(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:28,283 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:28,283 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=12, startTime=1733145447645; duration=0sec 2024-12-02T13:17:28,283 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:28,283 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:28,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145508384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145508384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145508385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T13:17:28,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:28,505 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T13:17:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:28,510 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:28,513 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:28,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145508595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145508595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145508595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:28,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/9bd63c34df71420990656eca73f8c6e8 2024-12-02T13:17:28,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T13:17:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:28,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/101a7630bb8b4bd299fb42bdc5d585bf is 50, key is test_row_0/B:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741898_1074 (size=12301) 2024-12-02T13:17:28,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/101a7630bb8b4bd299fb42bdc5d585bf 2024-12-02T13:17:28,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ec12ac8645a246edaeaf1bef1ec33b45 is 50, key is test_row_0/C:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741899_1075 (size=12301) 2024-12-02T13:17:28,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ec12ac8645a246edaeaf1bef1ec33b45 2024-12-02T13:17:28,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/9bd63c34df71420990656eca73f8c6e8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8 2024-12-02T13:17:28,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8, entries=200, sequenceid=298, filesize=14.4 K 2024-12-02T13:17:28,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/101a7630bb8b4bd299fb42bdc5d585bf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf 2024-12-02T13:17:28,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf, entries=150, sequenceid=298, filesize=12.0 K 2024-12-02T13:17:28,747 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T13:17:28,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/ec12ac8645a246edaeaf1bef1ec33b45 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45 2024-12-02T13:17:28,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45, entries=150, sequenceid=298, filesize=12.0 K 2024-12-02T13:17:28,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f8f9c71492391b16a7ca84f91364ebb5 in 538ms, sequenceid=298, compaction requested=true 2024-12-02T13:17:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:28,759 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:28,759 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:28,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:28,761 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44837 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:28,761 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:28,761 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:28,761 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:28,761 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,761 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,761 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a5da019a8b664a1ca91203f725e6c9d8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.6 K 2024-12-02T13:17:28,761 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/124112ffb9c441c1a90edeac045a4036, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=43.8 K 2024-12-02T13:17:28,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a5da019a8b664a1ca91203f725e6c9d8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:28,762 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 124112ffb9c441c1a90edeac045a4036, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:28,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b8851ed8282149cd842daec55ba8226c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733145446690 2024-12-02T13:17:28,762 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e63745357cce4d51adb9827e30624d30, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733145446690 2024-12-02T13:17:28,763 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 101a7630bb8b4bd299fb42bdc5d585bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:28,763 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd63c34df71420990656eca73f8c6e8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:28,775 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:28,775 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1d0663342e3c4166a5956269ffad6b83 is 50, key is test_row_0/B:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,787 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:28,787 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b986c3e3adcd4bc588e2fed091848402 is 50, key is test_row_0/A:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T13:17:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,822 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:28,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741900_1076 (size=13017) 2024-12-02T13:17:28,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/8bc34350423c48cba238f9b2ce70cb5d is 50, key is test_row_0/A:col10/1733145448259/Put/seqid=0 2024-12-02T13:17:28,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741901_1077 (size=13017) 2024-12-02T13:17:28,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741902_1078 (size=12301) 2024-12-02T13:17:28,855 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b986c3e3adcd4bc588e2fed091848402 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b986c3e3adcd4bc588e2fed091848402 2024-12-02T13:17:28,872 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into b986c3e3adcd4bc588e2fed091848402(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:28,872 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:28,872 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145448758; duration=0sec 2024-12-02T13:17:28,872 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:28,872 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:28,872 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:28,874 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:28,874 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:28,875 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:28,875 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c1c341caaaa0467ca6a66bc3a685e16f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.6 K 2024-12-02T13:17:28,875 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1c341caaaa0467ca6a66bc3a685e16f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733145446034 2024-12-02T13:17:28,876 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1480810a813748ca89b283da2fd2ed8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733145446690 2024-12-02T13:17:28,876 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec12ac8645a246edaeaf1bef1ec33b45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:28,902 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:28,906 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/b03ded206bac4fe9a086ef7bff349d2d is 50, key is test_row_0/C:col10/1733145448220/Put/seqid=0 2024-12-02T13:17:28,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:28,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741903_1079 (size=13017) 2024-12-02T13:17:28,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145508946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,958 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/b03ded206bac4fe9a086ef7bff349d2d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b03ded206bac4fe9a086ef7bff349d2d 2024-12-02T13:17:28,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145508956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145508956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:28,965 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into b03ded206bac4fe9a086ef7bff349d2d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:28,965 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:28,966 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145448759; duration=0sec 2024-12-02T13:17:28,966 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:28,966 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:29,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145509065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145509074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145509087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:29,237 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1d0663342e3c4166a5956269ffad6b83 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1d0663342e3c4166a5956269ffad6b83 2024-12-02T13:17:29,247 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 1d0663342e3c4166a5956269ffad6b83(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:29,247 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:29,247 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145448759; duration=0sec 2024-12-02T13:17:29,247 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:29,247 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:29,249 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/8bc34350423c48cba238f9b2ce70cb5d 2024-12-02T13:17:29,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/d9627e060e754bb9b834039710a6162f is 50, key is test_row_0/B:col10/1733145448259/Put/seqid=0 2024-12-02T13:17:29,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741904_1080 (size=12301) 2024-12-02T13:17:29,281 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/d9627e060e754bb9b834039710a6162f 2024-12-02T13:17:29,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145509284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145509284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bccfa3ed6c044e6bd9cd11f58ec1818 is 50, key is test_row_0/C:col10/1733145448259/Put/seqid=0 2024-12-02T13:17:29,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145509304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741905_1081 (size=12301) 2024-12-02T13:17:29,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57814 deadline: 1733145509374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,380 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57754 deadline: 1733145509404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,408 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145509604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145509604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145509613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:29,754 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bccfa3ed6c044e6bd9cd11f58ec1818 2024-12-02T13:17:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/8bc34350423c48cba238f9b2ce70cb5d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d 2024-12-02T13:17:29,772 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T13:17:29,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/d9627e060e754bb9b834039710a6162f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f 2024-12-02T13:17:29,784 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T13:17:29,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/0bccfa3ed6c044e6bd9cd11f58ec1818 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818 2024-12-02T13:17:29,798 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818, entries=150, sequenceid=312, filesize=12.0 K 2024-12-02T13:17:29,799 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for f8f9c71492391b16a7ca84f91364ebb5 in 977ms, sequenceid=312, compaction requested=false 2024-12-02T13:17:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T13:17:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T13:17:29,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T13:17:29,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2880 sec 2024-12-02T13:17:29,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3050 sec 2024-12-02T13:17:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:30,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:30,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d43a18cbd7fc4dd38448d0459da09f5c is 50, key is test_row_0/A:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:30,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145510137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145510144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145510144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741906_1082 (size=14741) 2024-12-02T13:17:30,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145510253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145510254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145510254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145510465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145510468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145510468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d43a18cbd7fc4dd38448d0459da09f5c 2024-12-02T13:17:30,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/098c4005770146ffb07d7a0a7e9ef3f1 is 50, key is test_row_0/B:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:30,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741907_1083 (size=12301) 2024-12-02T13:17:30,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/098c4005770146ffb07d7a0a7e9ef3f1 2024-12-02T13:17:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T13:17:30,654 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:30,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T13:17:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:30,658 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:30,659 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:30,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:30,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f80eb7e1a97547c29c4e623b8813f64d is 50, key is test_row_0/C:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:30,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741908_1084 (size=12301) 2024-12-02T13:17:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:30,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145510784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145510784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145510784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:30,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T13:17:30,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:30,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T13:17:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:30,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:30,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:31,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f80eb7e1a97547c29c4e623b8813f64d 2024-12-02T13:17:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/d43a18cbd7fc4dd38448d0459da09f5c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c 2024-12-02T13:17:31,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T13:17:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:31,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c, entries=200, sequenceid=337, filesize=14.4 K 2024-12-02T13:17:31,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/098c4005770146ffb07d7a0a7e9ef3f1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1 2024-12-02T13:17:31,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T13:17:31,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f80eb7e1a97547c29c4e623b8813f64d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d 2024-12-02T13:17:31,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d, entries=150, sequenceid=337, filesize=12.0 K 2024-12-02T13:17:31,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for f8f9c71492391b16a7ca84f91364ebb5 in 1051ms, sequenceid=337, compaction requested=true 2024-12-02T13:17:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:31,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:31,173 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:31,173 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:31,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:31,176 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:31,176 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:31,177 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,177 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b986c3e3adcd4bc588e2fed091848402, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=39.1 K 2024-12-02T13:17:31,177 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:31,177 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b986c3e3adcd4bc588e2fed091848402, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:31,177 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:31,177 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,178 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1d0663342e3c4166a5956269ffad6b83, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.7 K 2024-12-02T13:17:31,178 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d0663342e3c4166a5956269ffad6b83, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:31,178 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc34350423c48cba238f9b2ce70cb5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733145448256 2024-12-02T13:17:31,178 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d9627e060e754bb9b834039710a6162f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733145448256 2024-12-02T13:17:31,179 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d43a18cbd7fc4dd38448d0459da09f5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:31,179 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 098c4005770146ffb07d7a0a7e9ef3f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:31,207 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:31,208 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/97fbb7bd27ae4fb18648e4a8f201f946 is 50, key is test_row_0/B:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:31,212 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:31,212 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/837ce860bf48484e95a69873a75d21a3 is 50, key is test_row_0/A:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:31,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741909_1085 (size=13119) 2024-12-02T13:17:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741910_1086 (size=13119) 2024-12-02T13:17:31,247 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/97fbb7bd27ae4fb18648e4a8f201f946 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/97fbb7bd27ae4fb18648e4a8f201f946 2024-12-02T13:17:31,261 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 97fbb7bd27ae4fb18648e4a8f201f946(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:31,261 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:31,261 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145451173; duration=0sec 2024-12-02T13:17:31,261 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:31,261 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:31,262 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:31,263 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:31,264 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:31,264 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,264 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b03ded206bac4fe9a086ef7bff349d2d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.7 K 2024-12-02T13:17:31,265 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b03ded206bac4fe9a086ef7bff349d2d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145447879 2024-12-02T13:17:31,265 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bccfa3ed6c044e6bd9cd11f58ec1818, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733145448256 2024-12-02T13:17:31,265 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f80eb7e1a97547c29c4e623b8813f64d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:31,277 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:31,277 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/d6750ddd4e514207950bf10f67c1e8b2 is 50, key is test_row_0/C:col10/1733145448943/Put/seqid=0 2024-12-02T13:17:31,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T13:17:31,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:31,278 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:31,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b8de68b8a94f4ec7875217bd156272ef is 50, key is test_row_0/A:col10/1733145450124/Put/seqid=0 2024-12-02T13:17:31,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:31,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:31,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741911_1087 (size=13119) 2024-12-02T13:17:31,342 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/d6750ddd4e514207950bf10f67c1e8b2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/d6750ddd4e514207950bf10f67c1e8b2 2024-12-02T13:17:31,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741912_1088 (size=12301) 2024-12-02T13:17:31,349 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into d6750ddd4e514207950bf10f67c1e8b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:31,349 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:31,349 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145451173; duration=0sec 2024-12-02T13:17:31,350 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:31,350 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145511348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145511351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145451338, max timestamp=1733145451351Num uniques 859; 2024-12-02T13:17:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145451338, max timestamp=1733145451351Num uniques 859; 2024-12-02T13:17:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:31,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145451338, max timestamp=1733145451351Num uniques 859; 2024-12-02T13:17:31,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:31,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145511425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145511464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145511464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145511534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/837ce860bf48484e95a69873a75d21a3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/837ce860bf48484e95a69873a75d21a3 2024-12-02T13:17:31,654 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into 837ce860bf48484e95a69873a75d21a3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:31,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:31,655 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145451172; duration=0sec 2024-12-02T13:17:31,655 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:31,655 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:31,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145511674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145511674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145511744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,747 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b8de68b8a94f4ec7875217bd156272ef 2024-12-02T13:17:31,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/12eae83c4ffb4cc09c8b43af7f1f7181 is 50, key is test_row_0/B:col10/1733145450124/Put/seqid=0 2024-12-02T13:17:31,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741913_1089 (size=12301) 2024-12-02T13:17:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145511983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:31,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145511984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:31,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145512054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,163 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/12eae83c4ffb4cc09c8b43af7f1f7181 2024-12-02T13:17:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c2c08143c9e45dba8feb01d51ff9d31 is 50, key is test_row_0/C:col10/1733145450124/Put/seqid=0 2024-12-02T13:17:32,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741914_1090 (size=12301) 2024-12-02T13:17:32,188 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c2c08143c9e45dba8feb01d51ff9d31 2024-12-02T13:17:32,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/b8de68b8a94f4ec7875217bd156272ef as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef 2024-12-02T13:17:32,203 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef, entries=150, sequenceid=349, filesize=12.0 K 2024-12-02T13:17:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/12eae83c4ffb4cc09c8b43af7f1f7181 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181 2024-12-02T13:17:32,212 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181, entries=150, sequenceid=349, filesize=12.0 K 2024-12-02T13:17:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/5c2c08143c9e45dba8feb01d51ff9d31 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31 2024-12-02T13:17:32,219 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31, entries=150, sequenceid=349, filesize=12.0 K 2024-12-02T13:17:32,221 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f8f9c71492391b16a7ca84f91364ebb5 in 943ms, sequenceid=349, compaction requested=false 2024-12-02T13:17:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T13:17:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T13:17:32,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T13:17:32,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-12-02T13:17:32,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5720 sec 2024-12-02T13:17:32,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T13:17:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:32,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/918023b0714845f590b242586bdcbdc2 is 50, key is test_row_0/A:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145512527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145512529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741915_1091 (size=12301) 2024-12-02T13:17:32,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/918023b0714845f590b242586bdcbdc2 2024-12-02T13:17:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145512566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cd5013bdfb79454bbbbce132662d54f4 is 50, key is test_row_0/B:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:32,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741916_1092 (size=12301) 2024-12-02T13:17:32,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145512643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145512644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T13:17:32,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:32,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T13:17:32,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T13:17:32,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:32,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:32,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145512854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:32,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145512854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:32,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T13:17:32,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T13:17:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:32,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:32,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:32,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cd5013bdfb79454bbbbce132662d54f4 2024-12-02T13:17:33,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1929d96694384bca962c4283d9129b77 is 50, key is test_row_0/C:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:33,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741917_1093 (size=12301) 2024-12-02T13:17:33,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1929d96694384bca962c4283d9129b77 2024-12-02T13:17:33,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/918023b0714845f590b242586bdcbdc2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2 2024-12-02T13:17:33,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2, entries=150, sequenceid=388, filesize=12.0 K 2024-12-02T13:17:33,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/cd5013bdfb79454bbbbce132662d54f4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4 2024-12-02T13:17:33,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4, entries=150, sequenceid=388, filesize=12.0 K 2024-12-02T13:17:33,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/1929d96694384bca962c4283d9129b77 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77 2024-12-02T13:17:33,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77, entries=150, sequenceid=388, filesize=12.0 K 2024-12-02T13:17:33,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f8f9c71492391b16a7ca84f91364ebb5 in 595ms, sequenceid=388, compaction requested=true 2024-12-02T13:17:33,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:33,093 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:33,094 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:33,094 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:33,094 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:33,095 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/837ce860bf48484e95a69873a75d21a3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.8 K 2024-12-02T13:17:33,095 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837ce860bf48484e95a69873a75d21a3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:33,096 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8de68b8a94f4ec7875217bd156272ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733145450124 2024-12-02T13:17:33,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:33,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:33,096 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918023b0714845f590b242586bdcbdc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:33,096 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:33,098 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:33,098 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:33,098 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:33,098 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/97fbb7bd27ae4fb18648e4a8f201f946, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.8 K 2024-12-02T13:17:33,099 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 97fbb7bd27ae4fb18648e4a8f201f946, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:33,100 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 12eae83c4ffb4cc09c8b43af7f1f7181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733145450124 2024-12-02T13:17:33,101 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5013bdfb79454bbbbce132662d54f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T13:17:33,127 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:33,127 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/8b5c78230f604e0fb8a1be35e7bd1c27 is 50, key is test_row_0/A:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:33,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T13:17:33,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:33,130 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:33,135 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:33,135 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/de421b316e3542219b45c7edfe348a54 is 50, key is test_row_0/B:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:33,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/76e11c0e35c0448aab32bec7d879ff61 is 50, key is test_row_0/A:col10/1733145452527/Put/seqid=0 2024-12-02T13:17:33,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741919_1095 (size=13221) 2024-12-02T13:17:33,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:33,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741918_1094 (size=13221) 2024-12-02T13:17:33,179 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/de421b316e3542219b45c7edfe348a54 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/de421b316e3542219b45c7edfe348a54 2024-12-02T13:17:33,187 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into de421b316e3542219b45c7edfe348a54(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:33,187 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:33,187 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145453096; duration=0sec 2024-12-02T13:17:33,188 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:33,188 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:33,188 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:33,189 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:33,189 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:33,190 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:33,191 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/d6750ddd4e514207950bf10f67c1e8b2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.8 K 2024-12-02T13:17:33,192 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d6750ddd4e514207950bf10f67c1e8b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733145448943 2024-12-02T13:17:33,192 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c2c08143c9e45dba8feb01d51ff9d31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733145450124 2024-12-02T13:17:33,193 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1929d96694384bca962c4283d9129b77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741920_1096 (size=12301) 2024-12-02T13:17:33,203 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/76e11c0e35c0448aab32bec7d879ff61 2024-12-02T13:17:33,229 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#84 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:33,229 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/b0292db26f4f42c3a66618c52040f441 is 50, key is test_row_0/C:col10/1733145452494/Put/seqid=0 2024-12-02T13:17:33,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/47684977170c48b28183a9351d556b14 is 50, key is test_row_0/B:col10/1733145452527/Put/seqid=0 2024-12-02T13:17:33,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145453194, max timestamp=1733145453235Num uniques 859; 2024-12-02T13:17:33,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:17:33,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145453194, max timestamp=1733145453235Num uniques 859; 2024-12-02T13:17:33,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:17:33,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145453194, max timestamp=1733145453235Num uniques 859; 2024-12-02T13:17:33,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:17:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741921_1097 (size=13221) 2024-12-02T13:17:33,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741922_1098 (size=12301) 2024-12-02T13:17:33,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145513296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145513297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145513414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145513414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T13:17:33,581 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/8b5c78230f604e0fb8a1be35e7bd1c27 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8b5c78230f604e0fb8a1be35e7bd1c27 2024-12-02T13:17:33,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57782 deadline: 1733145513584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,588 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in f8f9c71492391b16a7ca84f91364ebb5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,591 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into 8b5c78230f604e0fb8a1be35e7bd1c27(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:33,591 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:33,591 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145453093; duration=0sec 2024-12-02T13:17:33,591 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:33,591 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:33,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145513624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145513625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,670 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/47684977170c48b28183a9351d556b14 2024-12-02T13:17:33,678 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/b0292db26f4f42c3a66618c52040f441 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b0292db26f4f42c3a66618c52040f441 2024-12-02T13:17:33,693 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into b0292db26f4f42c3a66618c52040f441(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:33,693 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:33,693 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145453097; duration=0sec 2024-12-02T13:17:33,693 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:33,693 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/4241a96afff946c2acf519167102d256 is 50, key is test_row_0/C:col10/1733145452527/Put/seqid=0 2024-12-02T13:17:33,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741923_1099 (size=12301) 2024-12-02T13:17:33,725 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/4241a96afff946c2acf519167102d256 2024-12-02T13:17:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/76e11c0e35c0448aab32bec7d879ff61 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61 2024-12-02T13:17:33,754 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61, entries=150, sequenceid=397, filesize=12.0 K 2024-12-02T13:17:33,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/47684977170c48b28183a9351d556b14 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14 2024-12-02T13:17:33,766 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14, entries=150, sequenceid=397, filesize=12.0 K 2024-12-02T13:17:33,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/4241a96afff946c2acf519167102d256 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256 2024-12-02T13:17:33,787 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256, entries=150, sequenceid=397, filesize=12.0 K 2024-12-02T13:17:33,793 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f8f9c71492391b16a7ca84f91364ebb5 in 664ms, sequenceid=397, compaction requested=false 2024-12-02T13:17:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:33,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T13:17:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T13:17:33,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T13:17:33,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 986 msec 2024-12-02T13:17:33,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 994 msec 2024-12-02T13:17:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:33,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:33,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T13:17:33,945 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:33,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:33,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/aaaed5d6c8784482a8639e8b11738265 is 50, key is test_row_0/A:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T13:17:33,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:33,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:33,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145513950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:33,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145513953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:33,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741924_1100 (size=14741) 2024-12-02T13:17:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:34,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145514064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145514064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:34,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145514273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145514274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/aaaed5d6c8784482a8639e8b11738265 2024-12-02T13:17:34,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/0dba980b0b0041089aa2ca7b236d6a4a is 50, key is test_row_0/B:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:34,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741925_1101 (size=12301) 2024-12-02T13:17:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/0dba980b0b0041089aa2ca7b236d6a4a 2024-12-02T13:17:34,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/7526a46ec4eb4258acd1717a23067b2c is 50, key is test_row_0/C:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:34,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741926_1102 (size=12301) 2024-12-02T13:17:34,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:34,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57726 deadline: 1733145514583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57762 deadline: 1733145514584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:34,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:17:34,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,782 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-02T13:17:34,782 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-02T13:17:34,782 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:34,782 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:34,782 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-02T13:17:34,782 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:34,786 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,786 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,786 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,786 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:34,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:34,786 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,787 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:34,790 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-02T13:17:34,790 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:34,790 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,790 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:34,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:34,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/7526a46ec4eb4258acd1717a23067b2c 2024-12-02T13:17:34,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/aaaed5d6c8784482a8639e8b11738265 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265 2024-12-02T13:17:34,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265, entries=200, sequenceid=437, filesize=14.4 K 2024-12-02T13:17:34,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/0dba980b0b0041089aa2ca7b236d6a4a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a 2024-12-02T13:17:34,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a, entries=150, sequenceid=437, filesize=12.0 K 2024-12-02T13:17:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/7526a46ec4eb4258acd1717a23067b2c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c 2024-12-02T13:17:34,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:34,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c, entries=150, sequenceid=437, filesize=12.0 K 2024-12-02T13:17:34,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for f8f9c71492391b16a7ca84f91364ebb5 in 939ms, sequenceid=437, compaction requested=true 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:34,875 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:34,875 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f9c71492391b16a7ca84f91364ebb5:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:34,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:34,876 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:34,876 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:34,876 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/A is initiating minor compaction (all files) 2024-12-02T13:17:34,876 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/B is initiating minor compaction (all files) 2024-12-02T13:17:34,876 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/B in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,876 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/A in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,877 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/de421b316e3542219b45c7edfe348a54, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.9 K 2024-12-02T13:17:34,877 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8b5c78230f604e0fb8a1be35e7bd1c27, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=39.3 K 2024-12-02T13:17:34,877 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5c78230f604e0fb8a1be35e7bd1c27, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:34,877 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting de421b316e3542219b45c7edfe348a54, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:34,878 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 47684977170c48b28183a9351d556b14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733145452508 2024-12-02T13:17:34,878 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e11c0e35c0448aab32bec7d879ff61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733145452508 2024-12-02T13:17:34,878 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dba980b0b0041089aa2ca7b236d6a4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733145453292 2024-12-02T13:17:34,878 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaaed5d6c8784482a8639e8b11738265, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733145453276 2024-12-02T13:17:34,888 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:34,889 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/51f59c30a25e4c089497d98c151d53ba is 50, key is test_row_0/B:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:34,890 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:34,890 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/f2ec42a1fd9a48fab6e80e5defbbdab8 is 50, key is test_row_0/A:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741928_1104 (size=13323) 2024-12-02T13:17:34,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741927_1103 (size=13323) 2024-12-02T13:17:34,905 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/f2ec42a1fd9a48fab6e80e5defbbdab8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/f2ec42a1fd9a48fab6e80e5defbbdab8 2024-12-02T13:17:34,909 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/A of f8f9c71492391b16a7ca84f91364ebb5 into f2ec42a1fd9a48fab6e80e5defbbdab8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:34,910 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:34,910 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/A, priority=13, startTime=1733145454875; duration=0sec 2024-12-02T13:17:34,910 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:34,910 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:A 2024-12-02T13:17:34,910 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:34,911 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:34,911 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f9c71492391b16a7ca84f91364ebb5/C is initiating minor compaction (all files) 2024-12-02T13:17:34,911 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f9c71492391b16a7ca84f91364ebb5/C in TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:34,911 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b0292db26f4f42c3a66618c52040f441, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp, totalSize=36.9 K 2024-12-02T13:17:34,913 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0292db26f4f42c3a66618c52040f441, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145451375 2024-12-02T13:17:34,914 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4241a96afff946c2acf519167102d256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1733145452508 2024-12-02T13:17:34,914 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7526a46ec4eb4258acd1717a23067b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733145453292 2024-12-02T13:17:34,921 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f9c71492391b16a7ca84f91364ebb5#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:34,922 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/7ea1921387ae4ffa9eae123087f55168 is 50, key is test_row_0/C:col10/1733145453292/Put/seqid=0 2024-12-02T13:17:34,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741929_1105 (size=13323) 2024-12-02T13:17:35,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T13:17:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:35,026 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T13:17:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:35,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/65bf1f7387a0489b85939f98ffaf2236 is 50, key is test_row_0/A:col10/1733145453944/Put/seqid=0 2024-12-02T13:17:35,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741930_1106 (size=12301) 2024-12-02T13:17:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:35,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. as already flushing 2024-12-02T13:17:35,095 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-02T13:17:35,095 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:35,095 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-02T13:17:35,095 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,095 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:35,095 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,095 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,095 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:35,095 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:35,311 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/51f59c30a25e4c089497d98c151d53ba as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/51f59c30a25e4c089497d98c151d53ba 2024-12-02T13:17:35,321 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/B of f8f9c71492391b16a7ca84f91364ebb5 into 51f59c30a25e4c089497d98c151d53ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:35,321 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:35,321 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/B, priority=13, startTime=1733145454875; duration=0sec 2024-12-02T13:17:35,321 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:35,321 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:B 2024-12-02T13:17:35,335 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/7ea1921387ae4ffa9eae123087f55168 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7ea1921387ae4ffa9eae123087f55168 2024-12-02T13:17:35,340 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f9c71492391b16a7ca84f91364ebb5/C of f8f9c71492391b16a7ca84f91364ebb5 into 7ea1921387ae4ffa9eae123087f55168(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:35,340 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:35,340 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., storeName=f8f9c71492391b16a7ca84f91364ebb5/C, priority=13, startTime=1733145454875; duration=0sec 2024-12-02T13:17:35,340 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:35,340 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f9c71492391b16a7ca84f91364ebb5:C 2024-12-02T13:17:35,436 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/65bf1f7387a0489b85939f98ffaf2236 2024-12-02T13:17:35,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1ee7950f661a4bfd932da5d0553d007c is 50, key is test_row_0/B:col10/1733145453944/Put/seqid=0 2024-12-02T13:17:35,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741931_1107 (size=12301) 2024-12-02T13:17:35,615 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-02T13:17:35,615 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:35,615 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,615 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:35,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:35,847 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1ee7950f661a4bfd932da5d0553d007c 2024-12-02T13:17:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f33edcdb58e24f5eb9f5b831108bc2df is 50, key is test_row_0/C:col10/1733145453944/Put/seqid=0 2024-12-02T13:17:35,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741932_1108 (size=12301) 2024-12-02T13:17:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:36,260 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f33edcdb58e24f5eb9f5b831108bc2df 2024-12-02T13:17:36,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/65bf1f7387a0489b85939f98ffaf2236 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/65bf1f7387a0489b85939f98ffaf2236 2024-12-02T13:17:36,269 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/65bf1f7387a0489b85939f98ffaf2236, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T13:17:36,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/1ee7950f661a4bfd932da5d0553d007c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1ee7950f661a4bfd932da5d0553d007c 2024-12-02T13:17:36,275 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1ee7950f661a4bfd932da5d0553d007c, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T13:17:36,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/f33edcdb58e24f5eb9f5b831108bc2df as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f33edcdb58e24f5eb9f5b831108bc2df 2024-12-02T13:17:36,280 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f33edcdb58e24f5eb9f5b831108bc2df, entries=150, sequenceid=446, filesize=12.0 K 2024-12-02T13:17:36,281 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=20.13 KB/20610 for f8f9c71492391b16a7ca84f91364ebb5 in 1255ms, sequenceid=446, compaction requested=false 2024-12-02T13:17:36,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f8f9c71492391b16a7ca84f91364ebb5: 2024-12-02T13:17:36,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:36,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T13:17:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T13:17:36,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T13:17:36,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3340 sec 2024-12-02T13:17:36,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3380 sec 2024-12-02T13:17:36,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:17:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T13:17:38,124 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:39,455 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-02T13:17:39,455 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:39,455 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,456 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:39,507 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-02T13:17:39,507 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:17:39,507 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,507 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:39,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:17:39,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-02T13:17:39,507 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2654 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2680 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1244 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3732 rows 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1256 2024-12-02T13:17:39,508 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3767 rows 2024-12-02T13:17:39,508 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:17:39,508 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:17:39,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:39,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:17:39,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:17:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:39,519 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145459519"}]},"ts":"1733145459519"} 2024-12-02T13:17:39,521 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:17:39,521 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:17:39,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:17:39,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, UNASSIGN}] 2024-12-02T13:17:39,527 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, UNASSIGN 2024-12-02T13:17:39,528 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f8f9c71492391b16a7ca84f91364ebb5, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:39,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, UNASSIGN because future has completed 2024-12-02T13:17:39,530 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:17:39,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:39,687 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:39,687 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing f8f9c71492391b16a7ca84f91364ebb5, disabling compactions & flushes 2024-12-02T13:17:39,688 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. after waiting 0 ms 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:39,688 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing f8f9c71492391b16a7ca84f91364ebb5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=A 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=B 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f9c71492391b16a7ca84f91364ebb5, store=C 2024-12-02T13:17:39,688 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:39,693 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/4ef3bd1b6fdc439b9bf3533f4d712552 is 50, key is test_row_0/A:col10/1733145459454/Put/seqid=0 2024-12-02T13:17:39,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741933_1109 (size=12301) 2024-12-02T13:17:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:40,097 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/4ef3bd1b6fdc439b9bf3533f4d712552 2024-12-02T13:17:40,105 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c7c0a483b65c4a3192953ab4adb6958e is 50, key is test_row_0/B:col10/1733145459454/Put/seqid=0 2024-12-02T13:17:40,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741934_1110 (size=12301) 2024-12-02T13:17:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:40,512 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c7c0a483b65c4a3192953ab4adb6958e 2024-12-02T13:17:40,519 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/338d4cf4af0a446ebcf9f69369a66eb6 is 50, key is test_row_0/C:col10/1733145459454/Put/seqid=0 2024-12-02T13:17:40,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741935_1111 (size=12301) 2024-12-02T13:17:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:40,923 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/338d4cf4af0a446ebcf9f69369a66eb6 2024-12-02T13:17:40,929 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/A/4ef3bd1b6fdc439b9bf3533f4d712552 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/4ef3bd1b6fdc439b9bf3533f4d712552 2024-12-02T13:17:40,934 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/4ef3bd1b6fdc439b9bf3533f4d712552, entries=150, sequenceid=456, filesize=12.0 K 2024-12-02T13:17:40,935 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/B/c7c0a483b65c4a3192953ab4adb6958e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c7c0a483b65c4a3192953ab4adb6958e 2024-12-02T13:17:40,940 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c7c0a483b65c4a3192953ab4adb6958e, entries=150, sequenceid=456, filesize=12.0 K 2024-12-02T13:17:40,941 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/.tmp/C/338d4cf4af0a446ebcf9f69369a66eb6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/338d4cf4af0a446ebcf9f69369a66eb6 2024-12-02T13:17:40,945 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/338d4cf4af0a446ebcf9f69369a66eb6, entries=150, sequenceid=456, filesize=12.0 K 2024-12-02T13:17:40,946 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8f9c71492391b16a7ca84f91364ebb5 in 1258ms, sequenceid=456, compaction requested=true 2024-12-02T13:17:40,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/df35e39d2cb54e5bae4214471b3e1080, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/a095d09c523944a59e4e3975d4ae2843, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2049060d94544261b29c4155e07bb59e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/61a84814840b48da834e1b4502f9268e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/124112ffb9c441c1a90edeac045a4036, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b986c3e3adcd4bc588e2fed091848402, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/837ce860bf48484e95a69873a75d21a3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8b5c78230f604e0fb8a1be35e7bd1c27, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265] to archive 2024-12-02T13:17:40,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:17:40,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b9089aebcbdb4dda8e3b0167682fa3f8 2024-12-02T13:17:40,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e58d33847d514b6c8dba7aa1ea2a477c 2024-12-02T13:17:40,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/df35e39d2cb54e5bae4214471b3e1080 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/df35e39d2cb54e5bae4214471b3e1080 2024-12-02T13:17:40,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/7ad3be706b9641e2a6e7df7682f261de 2024-12-02T13:17:40,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/0e5cd41995594d01ae001242aacd51cf 2024-12-02T13:17:40,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/fefd2703eae84d4eadd1f3ec6ed0994e 2024-12-02T13:17:40,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/a095d09c523944a59e4e3975d4ae2843 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/a095d09c523944a59e4e3975d4ae2843 2024-12-02T13:17:40,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/ae76bb2cbeda43378cbcac1ced5e8e63 2024-12-02T13:17:40,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/40763a6de7694d18b02c1126519dda90 2024-12-02T13:17:40,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/bbd3ea66277c4ac39a7bd8c5446eec1e 2024-12-02T13:17:40,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/272df6d3397c4a4b9cc75ab98f40ed4d 2024-12-02T13:17:40,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2049060d94544261b29c4155e07bb59e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2049060d94544261b29c4155e07bb59e 2024-12-02T13:17:40,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/2268b7ecc10b472b83bb0b9b1905d49d 2024-12-02T13:17:40,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d971660d77d145d78d6ee2482a9d4fd7 2024-12-02T13:17:40,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/61a84814840b48da834e1b4502f9268e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/61a84814840b48da834e1b4502f9268e 2024-12-02T13:17:40,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/953e5d21e3ca43948e49654f08d500b0 2024-12-02T13:17:40,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/59ec16ec9af24c13b76f7e4d3a212409 2024-12-02T13:17:40,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/124112ffb9c441c1a90edeac045a4036 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/124112ffb9c441c1a90edeac045a4036 2024-12-02T13:17:40,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/55a80b880a5f4bdb8d895fd8f862464b 2024-12-02T13:17:40,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/e63745357cce4d51adb9827e30624d30 2024-12-02T13:17:40,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/9bd63c34df71420990656eca73f8c6e8 2024-12-02T13:17:40,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b986c3e3adcd4bc588e2fed091848402 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b986c3e3adcd4bc588e2fed091848402 2024-12-02T13:17:40,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8bc34350423c48cba238f9b2ce70cb5d 2024-12-02T13:17:40,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/d43a18cbd7fc4dd38448d0459da09f5c 2024-12-02T13:17:40,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/837ce860bf48484e95a69873a75d21a3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/837ce860bf48484e95a69873a75d21a3 2024-12-02T13:17:40,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/b8de68b8a94f4ec7875217bd156272ef 2024-12-02T13:17:40,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8b5c78230f604e0fb8a1be35e7bd1c27 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/8b5c78230f604e0fb8a1be35e7bd1c27 2024-12-02T13:17:40,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/918023b0714845f590b242586bdcbdc2 2024-12-02T13:17:40,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/76e11c0e35c0448aab32bec7d879ff61 2024-12-02T13:17:40,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/aaaed5d6c8784482a8639e8b11738265 2024-12-02T13:17:41,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/2531723a944741ceb1471e3d43c9b3d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/024b873875c34e6cb14931d3b4bcf80b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0d217415bf224dd08e06ded6388ba944, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c80074203b164cf4844bee345b79a35c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a5da019a8b664a1ca91203f725e6c9d8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1d0663342e3c4166a5956269ffad6b83, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/97fbb7bd27ae4fb18648e4a8f201f946, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/de421b316e3542219b45c7edfe348a54, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a] to archive 2024-12-02T13:17:41,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:17:41,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1fcdf4e0354c4ec5aada046b410495ac 2024-12-02T13:17:41,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/3f35f3952227447fad7be326eaeaa310 2024-12-02T13:17:41,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/2531723a944741ceb1471e3d43c9b3d3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/2531723a944741ceb1471e3d43c9b3d3 2024-12-02T13:17:41,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e7ac60684ea34e409d4b3f54a647dc26 2024-12-02T13:17:41,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e17806754b06402da30c21c3c6c49256 2024-12-02T13:17:41,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/523a21bf93ec41f09b5f8233b11dc7c0 2024-12-02T13:17:41,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/024b873875c34e6cb14931d3b4bcf80b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/024b873875c34e6cb14931d3b4bcf80b 2024-12-02T13:17:41,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a97da89d27094c43af56abf3dc1108d3 2024-12-02T13:17:41,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/e6d55d57f96f4e71843c11541f3e1320 2024-12-02T13:17:41,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cea02bd8588742b095e40cbed4eab7da 2024-12-02T13:17:41,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0d217415bf224dd08e06ded6388ba944 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0d217415bf224dd08e06ded6388ba944 2024-12-02T13:17:41,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c211694e357d43338b56eb36467fb49d 2024-12-02T13:17:41,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c12f015bbbee4818bffe95829b959f56 2024-12-02T13:17:41,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c80074203b164cf4844bee345b79a35c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c80074203b164cf4844bee345b79a35c 2024-12-02T13:17:41,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/7fbdaf47c3664e6986fd4397eb490b88 2024-12-02T13:17:41,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f07092a356584c7eb4c84441d511dc62 2024-12-02T13:17:41,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/eeec56ca8c6a4764ab88b87d83821e51 2024-12-02T13:17:41,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a5da019a8b664a1ca91203f725e6c9d8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/a5da019a8b664a1ca91203f725e6c9d8 2024-12-02T13:17:41,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/f04ae371a52948b58d256a672e03f6e2 2024-12-02T13:17:41,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/b8851ed8282149cd842daec55ba8226c 2024-12-02T13:17:41,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1d0663342e3c4166a5956269ffad6b83 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1d0663342e3c4166a5956269ffad6b83 2024-12-02T13:17:41,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/101a7630bb8b4bd299fb42bdc5d585bf 2024-12-02T13:17:41,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/d9627e060e754bb9b834039710a6162f 2024-12-02T13:17:41,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/97fbb7bd27ae4fb18648e4a8f201f946 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/97fbb7bd27ae4fb18648e4a8f201f946 2024-12-02T13:17:41,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/098c4005770146ffb07d7a0a7e9ef3f1 2024-12-02T13:17:41,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/12eae83c4ffb4cc09c8b43af7f1f7181 2024-12-02T13:17:41,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/de421b316e3542219b45c7edfe348a54 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/de421b316e3542219b45c7edfe348a54 2024-12-02T13:17:41,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/cd5013bdfb79454bbbbce132662d54f4 2024-12-02T13:17:41,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/47684977170c48b28183a9351d556b14 2024-12-02T13:17:41,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/0dba980b0b0041089aa2ca7b236d6a4a 2024-12-02T13:17:41,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/583a75431f0f48faa8d1f2cd76d89ffb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4a4da4e4bba040cb98b1251203e5eded, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/14b009bf42d2496cb89545e17980d848, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/89dfc70e11234ea88c15f5144832f152, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c1c341caaaa0467ca6a66bc3a685e16f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b03ded206bac4fe9a086ef7bff349d2d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/d6750ddd4e514207950bf10f67c1e8b2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b0292db26f4f42c3a66618c52040f441, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c] to archive 2024-12-02T13:17:41,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:17:41,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/101ed2a20fad4e199d27869d313d7e10 2024-12-02T13:17:41,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/e71eb2ae3be34ef09c0f8e2fcb4a220c 2024-12-02T13:17:41,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/583a75431f0f48faa8d1f2cd76d89ffb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/583a75431f0f48faa8d1f2cd76d89ffb 2024-12-02T13:17:41,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/aabfd35bfeee4e17a9f2cad2c15faadc 2024-12-02T13:17:41,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/a95218356ac44d6a9f6355001be1371b 2024-12-02T13:17:41,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/eb8852c259b5482394cd0f75c0d88374 2024-12-02T13:17:41,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4a4da4e4bba040cb98b1251203e5eded to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4a4da4e4bba040cb98b1251203e5eded 2024-12-02T13:17:41,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ca9e42d6471c4b4d8e0e28f822e135b7 2024-12-02T13:17:41,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1d54bd69f73b44f181dc89675334654d 2024-12-02T13:17:41,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2c669de185a1487ba6adcaa0deea3c65 2024-12-02T13:17:41,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/14b009bf42d2496cb89545e17980d848 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/14b009bf42d2496cb89545e17980d848 2024-12-02T13:17:41,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c9dde7b4fcd54f05a0f5b65d4aa42eaf 2024-12-02T13:17:41,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c405070c2e349aaa436bc7d727ba6ba 2024-12-02T13:17:41,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/89dfc70e11234ea88c15f5144832f152 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/89dfc70e11234ea88c15f5144832f152 2024-12-02T13:17:41,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/04afeaa7d6164991be205d0f67161c0d 2024-12-02T13:17:41,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/2dbd98a4bc664ecaa4703de7b1c18f6d 2024-12-02T13:17:41,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ce63b4789f724f71975215451d2bdfa4 2024-12-02T13:17:41,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c1c341caaaa0467ca6a66bc3a685e16f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/c1c341caaaa0467ca6a66bc3a685e16f 2024-12-02T13:17:41,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bee3b8e149a447a94e84e9478fa681c 2024-12-02T13:17:41,067 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1480810a813748ca89b283da2fd2ed8e 2024-12-02T13:17:41,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b03ded206bac4fe9a086ef7bff349d2d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b03ded206bac4fe9a086ef7bff349d2d 2024-12-02T13:17:41,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/ec12ac8645a246edaeaf1bef1ec33b45 2024-12-02T13:17:41,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/0bccfa3ed6c044e6bd9cd11f58ec1818 2024-12-02T13:17:41,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/d6750ddd4e514207950bf10f67c1e8b2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/d6750ddd4e514207950bf10f67c1e8b2 2024-12-02T13:17:41,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f80eb7e1a97547c29c4e623b8813f64d 2024-12-02T13:17:41,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/5c2c08143c9e45dba8feb01d51ff9d31 2024-12-02T13:17:41,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b0292db26f4f42c3a66618c52040f441 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/b0292db26f4f42c3a66618c52040f441 2024-12-02T13:17:41,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/1929d96694384bca962c4283d9129b77 2024-12-02T13:17:41,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/4241a96afff946c2acf519167102d256 2024-12-02T13:17:41,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7526a46ec4eb4258acd1717a23067b2c 2024-12-02T13:17:41,082 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/recovered.edits/459.seqid, newMaxSeqId=459, maxSeqId=1 2024-12-02T13:17:41,082 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. 2024-12-02T13:17:41,083 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for f8f9c71492391b16a7ca84f91364ebb5: Waiting for close lock at 1733145459688Running coprocessor pre-close hooks at 1733145459688Disabling compacts and flushes for region at 1733145459688Disabling writes for close at 1733145459688Obtaining lock to block concurrent updates at 1733145459688Preparing flush snapshotting stores in f8f9c71492391b16a7ca84f91364ebb5 at 1733145459688Finished memstore snapshotting TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733145459688Flushing stores of TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5. at 1733145459689 (+1 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/A: creating writer at 1733145459689Flushing f8f9c71492391b16a7ca84f91364ebb5/A: appending metadata at 1733145459692 (+3 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/A: closing flushed file at 1733145459692Flushing f8f9c71492391b16a7ca84f91364ebb5/B: creating writer at 1733145460102 (+410 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/B: appending metadata at 1733145460105 (+3 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/B: closing flushed file at 1733145460105Flushing f8f9c71492391b16a7ca84f91364ebb5/C: creating writer at 1733145460516 (+411 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/C: appending metadata at 1733145460519 (+3 ms)Flushing f8f9c71492391b16a7ca84f91364ebb5/C: closing flushed file at 1733145460519Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1817d5f2: reopening flushed file at 1733145460928 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5745775b: reopening flushed file at 1733145460934 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6b8698: reopening flushed file at 1733145460940 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8f9c71492391b16a7ca84f91364ebb5 in 1258ms, sequenceid=456, compaction requested=true at 1733145460946 (+6 ms)Writing region close event to WAL at 1733145461079 (+133 ms)Running coprocessor post-close hooks at 1733145461082 (+3 ms)Closed at 1733145461082 2024-12-02T13:17:41,085 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:41,085 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f8f9c71492391b16a7ca84f91364ebb5, regionState=CLOSED 2024-12-02T13:17:41,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:17:41,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T13:17:41,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure f8f9c71492391b16a7ca84f91364ebb5, server=06e3e18f2515,45693,1733145428280 in 1.5570 sec 2024-12-02T13:17:41,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-02T13:17:41,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f9c71492391b16a7ca84f91364ebb5, UNASSIGN in 1.5630 sec 2024-12-02T13:17:41,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T13:17:41,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5700 sec 2024-12-02T13:17:41,094 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145461094"}]},"ts":"1733145461094"} 2024-12-02T13:17:41,095 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:17:41,095 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:17:41,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5820 sec 2024-12-02T13:17:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T13:17:41,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:41,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:17:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T13:17:41,683 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,687 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:41,691 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/recovered.edits] 2024-12-02T13:17:41,693 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/4ef3bd1b6fdc439b9bf3533f4d712552 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/4ef3bd1b6fdc439b9bf3533f4d712552 2024-12-02T13:17:41,695 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/65bf1f7387a0489b85939f98ffaf2236 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/65bf1f7387a0489b85939f98ffaf2236 2024-12-02T13:17:41,696 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/f2ec42a1fd9a48fab6e80e5defbbdab8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/A/f2ec42a1fd9a48fab6e80e5defbbdab8 2024-12-02T13:17:41,698 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1ee7950f661a4bfd932da5d0553d007c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/1ee7950f661a4bfd932da5d0553d007c 2024-12-02T13:17:41,699 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/51f59c30a25e4c089497d98c151d53ba to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/51f59c30a25e4c089497d98c151d53ba 2024-12-02T13:17:41,700 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c7c0a483b65c4a3192953ab4adb6958e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/B/c7c0a483b65c4a3192953ab4adb6958e 2024-12-02T13:17:41,702 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/338d4cf4af0a446ebcf9f69369a66eb6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/338d4cf4af0a446ebcf9f69369a66eb6 2024-12-02T13:17:41,703 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7ea1921387ae4ffa9eae123087f55168 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/7ea1921387ae4ffa9eae123087f55168 2024-12-02T13:17:41,704 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f33edcdb58e24f5eb9f5b831108bc2df to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/C/f33edcdb58e24f5eb9f5b831108bc2df 2024-12-02T13:17:41,706 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/recovered.edits/459.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5/recovered.edits/459.seqid 2024-12-02T13:17:41,707 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f9c71492391b16a7ca84f91364ebb5 2024-12-02T13:17:41,707 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:17:41,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T13:17:41,720 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:17:41,747 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:17:41,748 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,748 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:17:41,748 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145461748"}]},"ts":"9223372036854775807"} 2024-12-02T13:17:41,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:17:41,753 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f8f9c71492391b16a7ca84f91364ebb5, NAME => 'TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:17:41,754 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:17:41,754 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145461754"}]},"ts":"9223372036854775807"} 2024-12-02T13:17:41,757 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:17:41,758 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-12-02T13:17:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T13:17:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:17:41,794 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:41,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;06e3e18f2515:45693-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_740994727_22 at /127.0.0.1:44190 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 264) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1806 (was 2587) 2024-12-02T13:17:41,811 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=1806 2024-12-02T13:17:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:17:41,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:17:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:41,815 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:17:41,815 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:41,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-02T13:17:41,816 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:17:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:41,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741936_1112 (size=1061) 2024-12-02T13:17:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:42,223 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:17:42,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741937_1113 (size=53) 2024-12-02T13:17:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:42,629 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:42,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f8f6542858c1817d9eb99a42a221bf04, disabling compactions & flushes 2024-12-02T13:17:42,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. after waiting 0 ms 2024-12-02T13:17:42,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,630 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,630 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8f6542858c1817d9eb99a42a221bf04: Waiting for close lock at 1733145462630Disabling compacts and flushes for region at 1733145462630Disabling writes for close at 1733145462630Writing region close event to WAL at 1733145462630Closed at 1733145462630 2024-12-02T13:17:42,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:17:42,631 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145462631"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145462631"}]},"ts":"1733145462631"} 2024-12-02T13:17:42,633 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:17:42,634 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:17:42,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145462634"}]},"ts":"1733145462634"} 2024-12-02T13:17:42,635 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:17:42,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, ASSIGN}] 2024-12-02T13:17:42,637 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, ASSIGN 2024-12-02T13:17:42,638 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:17:42,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:42,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, ASSIGN because future has completed 2024-12-02T13:17:42,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:42,945 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,946 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:17:42,946 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,946 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:42,946 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,946 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,947 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,949 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:42,950 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName A 2024-12-02T13:17:42,950 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:42,950 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:42,950 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,952 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:42,952 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName B 2024-12-02T13:17:42,952 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:42,952 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:42,953 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:42,954 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:42,954 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName C 2024-12-02T13:17:42,954 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:42,954 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:42,954 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,955 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,955 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,955 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,956 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,956 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,957 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:17:42,958 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,960 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:17:42,960 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened f8f6542858c1817d9eb99a42a221bf04; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69899038, jitterRate=0.04157683253288269}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:17:42,961 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:42,961 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for f8f6542858c1817d9eb99a42a221bf04: Running coprocessor pre-open hook at 1733145462946Writing region info on filesystem at 1733145462946Initializing all the Stores at 1733145462947 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145462947Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145462947Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145462947Cleaning up temporary data from old regions at 1733145462956 (+9 ms)Running coprocessor post-open hooks at 1733145462961 (+5 ms)Region opened successfully at 1733145462961 2024-12-02T13:17:42,962 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., pid=32, masterSystemTime=1733145462942 2024-12-02T13:17:42,965 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,965 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:42,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:42,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:17:42,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-02T13:17:42,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 in 178 msec 2024-12-02T13:17:42,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-02T13:17:42,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, ASSIGN in 336 msec 2024-12-02T13:17:42,975 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:17:42,975 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145462975"}]},"ts":"1733145462975"} 2024-12-02T13:17:42,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:17:42,977 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:17:42,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-12-02T13:17:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-02T13:17:43,964 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:43,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:43,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:43,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:43,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66120160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:43,968 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16870e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:43,969 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:43,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:43,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:17:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:17:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43c3c82e 2024-12-02T13:17:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:17:43,974 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:17:43,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:17:43,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:17:43,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T13:17:43,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741938_1114 (size=1097) 2024-12-02T13:17:44,397 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T13:17:44,397 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T13:17:44,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:17:44,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE}] 2024-12-02T13:17:44,407 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE 2024-12-02T13:17:44,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:44,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE because future has completed 2024-12-02T13:17:44,409 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:17:44,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:44,562 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,562 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:17:44,562 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing f8f6542858c1817d9eb99a42a221bf04, disabling compactions & flushes 2024-12-02T13:17:44,562 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,562 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,562 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. after waiting 0 ms 2024-12-02T13:17:44,562 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,566 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T13:17:44,566 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,566 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for f8f6542858c1817d9eb99a42a221bf04: Waiting for close lock at 1733145464562Running coprocessor pre-close hooks at 1733145464562Disabling compacts and flushes for region at 1733145464562Disabling writes for close at 1733145464562Writing region close event to WAL at 1733145464562Running coprocessor post-close hooks at 1733145464566 (+4 ms)Closed at 1733145464566 2024-12-02T13:17:44,566 WARN [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: f8f6542858c1817d9eb99a42a221bf04 to self. 2024-12-02T13:17:44,568 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,569 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=CLOSED 2024-12-02T13:17:44,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:17:44,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T13:17:44,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 in 162 msec 2024-12-02T13:17:44,573 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE; state=CLOSED, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=true 2024-12-02T13:17:44,724 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:44,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE because future has completed 2024-12-02T13:17:44,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:17:44,881 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,881 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:17:44,881 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,882 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:17:44,882 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,882 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,884 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,885 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:44,890 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName A 2024-12-02T13:17:44,892 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:44,892 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:44,893 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,893 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:44,893 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName B 2024-12-02T13:17:44,894 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:44,894 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:44,894 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,894 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:17:44,895 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f6542858c1817d9eb99a42a221bf04 columnFamilyName C 2024-12-02T13:17:44,895 DEBUG [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:44,895 INFO [StoreOpener-f8f6542858c1817d9eb99a42a221bf04-1 {}] regionserver.HStore(327): Store=f8f6542858c1817d9eb99a42a221bf04/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:17:44,895 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,895 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,896 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,897 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,897 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,897 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,898 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:17:44,899 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,900 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened f8f6542858c1817d9eb99a42a221bf04; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62783089, jitterRate=-0.06445907056331635}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:17:44,900 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,902 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for f8f6542858c1817d9eb99a42a221bf04: Running coprocessor pre-open hook at 1733145464882Writing region info on filesystem at 1733145464882Initializing all the Stores at 1733145464882Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145464882Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145464883 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145464883Cleaning up temporary data from old regions at 1733145464897 (+14 ms)Running coprocessor post-open hooks at 1733145464900 (+3 ms)Region opened successfully at 1733145464902 (+2 ms) 2024-12-02T13:17:44,902 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., pid=37, masterSystemTime=1733145464878 2024-12-02T13:17:44,904 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,904 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:44,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=OPEN, openSeqNum=5, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:44,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:17:44,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-02T13:17:44,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 in 181 msec 2024-12-02T13:17:44,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-02T13:17:44,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, REOPEN/MOVE in 503 msec 2024-12-02T13:17:44,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-02T13:17:44,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-12-02T13:17:44,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 935 msec 2024-12-02T13:17:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T13:17:44,919 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:44,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,921 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7ff182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78276902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,926 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@579718f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,928 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c983234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,930 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672baf8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c25f38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b2c4cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273fc4fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1c5c78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,940 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5663af90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,942 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b7a950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,944 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3842fe16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,946 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598a1cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,948 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c50e218, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,950 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74023b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3d25d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:17:44,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:17:44,954 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@880a0c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:17:44,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:17:44,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:17:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7405a174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:17:44,957 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,957 DEBUG [Thread-595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,957 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,958 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:44,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:44,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:44,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:44,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:44,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:44,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:44,976 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:44,977 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:44,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:44,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,981 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:44,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:44,992 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:44,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:44,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:45,001 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:45,003 DEBUG [Thread-591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:45,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:45,013 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:45,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:45,033 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:17:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:17:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:17:45,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:17:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:17:45,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026f952a4b63bf4056af286912e6e44387_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145464972/Put/seqid=0 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145465031, max timestamp=1733145465050Num uniques 859; 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145465031, max timestamp=1733145465050Num uniques 859; 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145465031, max timestamp=1733145465050Num uniques 859; 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:45,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:45,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741939_1115 (size=12154) 2024-12-02T13:17:45,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145525093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145525095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145525098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145525098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145525115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145525224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145525225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145525225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145525225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145525234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:45,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145525445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145525445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145525445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145525445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145525456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:45,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026f952a4b63bf4056af286912e6e44387_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f952a4b63bf4056af286912e6e44387_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:45,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1d7278ce799d4bcca3f286e0dffb2052, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:45,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1d7278ce799d4bcca3f286e0dffb2052 is 175, key is test_row_0/A:col10/1733145464972/Put/seqid=0 2024-12-02T13:17:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741940_1116 (size=30955) 2024-12-02T13:17:45,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:45,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145525755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145525756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145525756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145525764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145525769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:45,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:45,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:45,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:45,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1d7278ce799d4bcca3f286e0dffb2052 2024-12-02T13:17:46,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/557d2eaf16214fcca09d5e7fcaf2a313 is 50, key is test_row_0/B:col10/1733145464972/Put/seqid=0 2024-12-02T13:17:46,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:46,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:46,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741941_1117 (size=12001) 2024-12-02T13:17:46,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/557d2eaf16214fcca09d5e7fcaf2a313 2024-12-02T13:17:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:46,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1c0e77fd269f4db68094ae9e14b13a18 is 50, key is test_row_0/C:col10/1733145464972/Put/seqid=0 2024-12-02T13:17:46,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741942_1118 (size=12001) 2024-12-02T13:17:46,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145526274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145526275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:46,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145526275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:46,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145526284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145526284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:46,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:46,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:46,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:46,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1c0e77fd269f4db68094ae9e14b13a18 2024-12-02T13:17:46,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1d7278ce799d4bcca3f286e0dffb2052 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052 2024-12-02T13:17:46,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052, entries=150, sequenceid=14, filesize=30.2 K 2024-12-02T13:17:46,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/557d2eaf16214fcca09d5e7fcaf2a313 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313 2024-12-02T13:17:46,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T13:17:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1c0e77fd269f4db68094ae9e14b13a18 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18 2024-12-02T13:17:46,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18, entries=150, sequenceid=14, filesize=11.7 K 2024-12-02T13:17:46,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8f6542858c1817d9eb99a42a221bf04 in 1649ms, sequenceid=14, compaction requested=false 2024-12-02T13:17:46,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:46,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:17:46,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-02T13:17:46,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:46,682 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:46,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d5a52ae988184dceab9050cd9e1f47da_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145465090/Put/seqid=0 2024-12-02T13:17:46,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741943_1119 (size=12154) 2024-12-02T13:17:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:46,773 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d5a52ae988184dceab9050cd9e1f47da_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d5a52ae988184dceab9050cd9e1f47da_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3fc2e2de07a74ca3a663f26442fd7ca2, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3fc2e2de07a74ca3a663f26442fd7ca2 is 175, key is test_row_0/A:col10/1733145465090/Put/seqid=0 2024-12-02T13:17:46,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741944_1120 (size=30955) 2024-12-02T13:17:46,793 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3fc2e2de07a74ca3a663f26442fd7ca2 2024-12-02T13:17:46,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a3a28eb2753d49e89bec26f8cbeb9bcc is 50, key is test_row_0/B:col10/1733145465090/Put/seqid=0 2024-12-02T13:17:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741945_1121 (size=12001) 2024-12-02T13:17:46,850 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a3a28eb2753d49e89bec26f8cbeb9bcc 2024-12-02T13:17:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/5e8cd205dd144ea8ba7ab72c73d104ae is 50, key is test_row_0/C:col10/1733145465090/Put/seqid=0 2024-12-02T13:17:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741946_1122 (size=12001) 2024-12-02T13:17:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:47,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145527302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145527302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145527304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,308 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:47,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145527311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145527311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,323 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/5e8cd205dd144ea8ba7ab72c73d104ae 2024-12-02T13:17:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3fc2e2de07a74ca3a663f26442fd7ca2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2 2024-12-02T13:17:47,352 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2, entries=150, sequenceid=51, filesize=30.2 K 2024-12-02T13:17:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a3a28eb2753d49e89bec26f8cbeb9bcc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc 2024-12-02T13:17:47,372 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T13:17:47,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/5e8cd205dd144ea8ba7ab72c73d104ae as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae 2024-12-02T13:17:47,386 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae, entries=150, sequenceid=51, filesize=11.7 K 2024-12-02T13:17:47,390 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f8f6542858c1817d9eb99a42a221bf04 in 707ms, sequenceid=51, compaction requested=false 2024-12-02T13:17:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:47,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-02T13:17:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-02T13:17:47,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-02T13:17:47,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4130 sec 2024-12-02T13:17:47,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.4200 sec 2024-12-02T13:17:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:47,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:47,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025556968d82104987b1090bc4ace3e73c_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:47,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741947_1123 (size=12154) 2024-12-02T13:17:47,471 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:47,480 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025556968d82104987b1090bc4ace3e73c_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025556968d82104987b1090bc4ace3e73c_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:47,481 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/28ed30c4bb0546708ed23266bd5617de, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:47,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/28ed30c4bb0546708ed23266bd5617de is 175, key is test_row_0/A:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:47,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145527462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145527471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145527473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145527481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741948_1124 (size=30955) 2024-12-02T13:17:47,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/28ed30c4bb0546708ed23266bd5617de 2024-12-02T13:17:47,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 is 50, key is test_row_0/B:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:47,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741949_1125 (size=12001) 2024-12-02T13:17:47,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 2024-12-02T13:17:47,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f2d7cd3432c14dcea400ad944e6a4acf is 50, key is test_row_0/C:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:47,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741950_1126 (size=12001) 2024-12-02T13:17:47,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145527595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145527595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145527595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145527597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145527803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145527804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145527804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145527814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:47,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f2d7cd3432c14dcea400ad944e6a4acf 2024-12-02T13:17:48,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/28ed30c4bb0546708ed23266bd5617de as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de 2024-12-02T13:17:48,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de, entries=150, sequenceid=63, filesize=30.2 K 2024-12-02T13:17:48,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 2024-12-02T13:17:48,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2, entries=150, sequenceid=63, filesize=11.7 K 2024-12-02T13:17:48,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f2d7cd3432c14dcea400ad944e6a4acf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf 2024-12-02T13:17:48,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf, entries=150, sequenceid=63, filesize=11.7 K 2024-12-02T13:17:48,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f8f6542858c1817d9eb99a42a221bf04 in 621ms, sequenceid=63, compaction requested=true 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:48,041 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:48,041 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:48,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:48,042 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:48,042 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:48,042 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:48,042 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:48,042 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:48,042 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:48,042 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=90.7 K 2024-12-02T13:17:48,042 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.2 K 2024-12-02T13:17:48,043 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 557d2eaf16214fcca09d5e7fcaf2a313, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733145464968 2024-12-02T13:17:48,043 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:48,043 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de] 2024-12-02T13:17:48,043 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a28eb2753d49e89bec26f8cbeb9bcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733145465090 2024-12-02T13:17:48,043 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7278ce799d4bcca3f286e0dffb2052, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733145464968 2024-12-02T13:17:48,044 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8655b8483bbb42b9ae6c3f4fdd7c65b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:48,044 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc2e2de07a74ca3a663f26442fd7ca2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733145465090 2024-12-02T13:17:48,044 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ed30c4bb0546708ed23266bd5617de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:48,066 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:48,069 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:48,070 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/2c4ffdee6d7f4f2d9972f2f277296fdc is 50, key is test_row_0/B:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:48,090 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120279daac7e5fa349f2bb7f1b14a635fba6_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:48,100 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120279daac7e5fa349f2bb7f1b14a635fba6_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:48,100 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120279daac7e5fa349f2bb7f1b14a635fba6_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:48,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145467465, max timestamp=1733145468118Num uniques 859; 2024-12-02T13:17:48,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:48,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145467465, max timestamp=1733145468118Num uniques 859; 2024-12-02T13:17:48,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:48,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-02T13:17:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:48,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145467465, max timestamp=1733145468121Num uniques 859; 2024-12-02T13:17:48,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:48,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741951_1127 (size=12104) 2024-12-02T13:17:48,176 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/2c4ffdee6d7f4f2d9972f2f277296fdc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/2c4ffdee6d7f4f2d9972f2f277296fdc 2024-12-02T13:17:48,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145468147, max timestamp=1733145468176Num uniques 859; 2024-12-02T13:17:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145468147, max timestamp=1733145468176Num uniques 859; 2024-12-02T13:17:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145468147, max timestamp=1733145468176Num uniques 859; 2024-12-02T13:17:48,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:48,190 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into 2c4ffdee6d7f4f2d9972f2f277296fdc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:48,190 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:48,191 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=13, startTime=1733145468041; duration=0sec 2024-12-02T13:17:48,191 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:48,191 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:48,191 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:48,194 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:48,194 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:48,194 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:48,194 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.2 K 2024-12-02T13:17:48,195 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0e77fd269f4db68094ae9e14b13a18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733145464968 2024-12-02T13:17:48,195 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8cd205dd144ea8ba7ab72c73d104ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733145465090 2024-12-02T13:17:48,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d7cd3432c14dcea400ad944e6a4acf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:48,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741952_1128 (size=4469) 2024-12-02T13:17:48,199 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#108 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:48,200 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c4b07b3cb40049fc8333ec60eae614be is 175, key is test_row_0/A:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:48,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020314e060208d4fa8acc25b31c46fb1c6_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145468121/Put/seqid=0 2024-12-02T13:17:48,219 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:48,220 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/ea57b015324946adbf6f4dc88c1c237b is 50, key is test_row_0/C:col10/1733145467419/Put/seqid=0 2024-12-02T13:17:48,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733145468223, max timestamp=1733145468240Num uniques 1068; 2024-12-02T13:17:48,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T13:17:48,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733145468223, max timestamp=1733145468240Num uniques 1068; 2024-12-02T13:17:48,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733145468223, max timestamp=1733145468240Num uniques 1068; 2024-12-02T13:17:48,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T13:17:48,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T13:17:48,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741953_1129 (size=31058) 2024-12-02T13:17:48,267 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c4b07b3cb40049fc8333ec60eae614be as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be 2024-12-02T13:17:48,274 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into c4b07b3cb40049fc8333ec60eae614be(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:48,274 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:48,274 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=13, startTime=1733145468041; duration=0sec 2024-12-02T13:17:48,275 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:48,275 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:48,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733145468267, max timestamp=1733145468296Num uniques 1127; 2024-12-02T13:17:48,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733145468267, max timestamp=1733145468296Num uniques 1127; 2024-12-02T13:17:48,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-02T13:17:48,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-12-02T13:17:48,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733145468279, max timestamp=1733145468296Num uniques 1127; 2024-12-02T13:17:48,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=6870, suffix heap size=145176,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=150 2024-12-02T13:17:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741954_1130 (size=12154) 2024-12-02T13:17:48,307 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:48,313 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020314e060208d4fa8acc25b31c46fb1c6_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020314e060208d4fa8acc25b31c46fb1c6_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:48,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/9064544987734d6988c2dd7ab3eb8785, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:48,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/9064544987734d6988c2dd7ab3eb8785 is 175, key is test_row_0/A:col10/1733145468121/Put/seqid=0 2024-12-02T13:17:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741955_1131 (size=12104) 2024-12-02T13:17:48,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733145468348, max timestamp=1733145468360Num uniques 1086; 2024-12-02T13:17:48,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T13:17:48,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733145468348, max timestamp=1733145468360Num uniques 1086; 2024-12-02T13:17:48,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-02T13:17:48,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733145468348, max timestamp=1733145468360Num uniques 1009; 2024-12-02T13:17:48,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-12-02T13:17:48,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741956_1132 (size=30955) 2024-12-02T13:17:48,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/9064544987734d6988c2dd7ab3eb8785 2024-12-02T13:17:48,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/570ea49f38de4e318690b156595c20dd is 50, key is test_row_0/B:col10/1733145468121/Put/seqid=0 2024-12-02T13:17:48,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145468384, max timestamp=1733145468410Num uniques 1095; 2024-12-02T13:17:48,414 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T13:17:48,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145468384, max timestamp=1733145468413Num uniques 1095; 2024-12-02T13:17:48,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T13:17:48,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145468384, max timestamp=1733145468413Num uniques 1095; 2024-12-02T13:17:48,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-02T13:17:48,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741957_1133 (size=12001) 2024-12-02T13:17:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145528455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145528458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145528459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145528464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145528574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145528574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145528574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145528574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,745 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/ea57b015324946adbf6f4dc88c1c237b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ea57b015324946adbf6f4dc88c1c237b 2024-12-02T13:17:48,750 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into ea57b015324946adbf6f4dc88c1c237b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:48,750 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:48,750 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=13, startTime=1733145468041; duration=0sec 2024-12-02T13:17:48,750 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:48,750 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:48,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145528784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145528784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145528784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145528784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:48,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/570ea49f38de4e318690b156595c20dd 2024-12-02T13:17:48,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/608cafcdb68c4ef1813bdf163ba09ed1 is 50, key is test_row_0/C:col10/1733145468121/Put/seqid=0 2024-12-02T13:17:48,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741958_1134 (size=12001) 2024-12-02T13:17:48,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/608cafcdb68c4ef1813bdf163ba09ed1 2024-12-02T13:17:48,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/9064544987734d6988c2dd7ab3eb8785 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785 2024-12-02T13:17:48,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785, entries=150, sequenceid=85, filesize=30.2 K 2024-12-02T13:17:48,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/570ea49f38de4e318690b156595c20dd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd 2024-12-02T13:17:48,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd, entries=150, sequenceid=85, filesize=11.7 K 2024-12-02T13:17:48,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/608cafcdb68c4ef1813bdf163ba09ed1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1 2024-12-02T13:17:48,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1, entries=150, sequenceid=85, filesize=11.7 K 2024-12-02T13:17:48,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for f8f6542858c1817d9eb99a42a221bf04 in 805ms, sequenceid=85, compaction requested=false 2024-12-02T13:17:48,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:49,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-12-02T13:17:49,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733145468447, max timestamp=1733145469099Num uniques 1154; 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733145468447, max timestamp=1733145469099Num uniques 1154; 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733145468447, max timestamp=1733145469099Num uniques 1154; 2024-12-02T13:17:49,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-02T13:17:49,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:49,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T13:17:49,134 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:49,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-02T13:17:49,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bc44a277a23f4e82b9cbe5b4a4722e46_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:49,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:49,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:49,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:49,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733145469132, max timestamp=1733145469153Num uniques 859; 2024-12-02T13:17:49,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733145469132, max timestamp=1733145469153Num uniques 859; 2024-12-02T13:17:49,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T13:17:49,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T13:17:49,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733145469132, max timestamp=1733145469153Num uniques 859; 2024-12-02T13:17:49,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T13:17:49,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741959_1135 (size=12304) 2024-12-02T13:17:49,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:49,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bc44a277a23f4e82b9cbe5b4a4722e46_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bc44a277a23f4e82b9cbe5b4a4722e46_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:49,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/37fbfa9ed5a4490590fe9554e8cb1cc5, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:49,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/37fbfa9ed5a4490590fe9554e8cb1cc5 is 175, key is test_row_0/A:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145529200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145529200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145529215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145529215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741960_1136 (size=31105) 2024-12-02T13:17:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:49,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T13:17:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145529324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145529324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,341 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T13:17:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:49,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145529534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145529534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145529544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145529545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T13:17:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:49,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/37fbfa9ed5a4490590fe9554e8cb1cc5 2024-12-02T13:17:49,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/44ae809c4d92402f8926d0310671aa4b is 50, key is test_row_0/B:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741961_1137 (size=12151) 2024-12-02T13:17:49,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/44ae809c4d92402f8926d0310671aa4b 2024-12-02T13:17:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1a6adb332be44aecbe811ed7344d38dd is 50, key is test_row_0/C:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741962_1138 (size=12151) 2024-12-02T13:17:49,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1a6adb332be44aecbe811ed7344d38dd 2024-12-02T13:17:49,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/37fbfa9ed5a4490590fe9554e8cb1cc5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5 2024-12-02T13:17:49,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5, entries=150, sequenceid=196, filesize=30.4 K 2024-12-02T13:17:49,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/44ae809c4d92402f8926d0310671aa4b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b 2024-12-02T13:17:49,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b, entries=150, sequenceid=196, filesize=11.9 K 2024-12-02T13:17:49,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1a6adb332be44aecbe811ed7344d38dd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd 2024-12-02T13:17:49,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd, entries=150, sequenceid=196, filesize=11.9 K 2024-12-02T13:17:49,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=161.02 KB/164880 for f8f6542858c1817d9eb99a42a221bf04 in 648ms, sequenceid=196, compaction requested=true 2024-12-02T13:17:49,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:49,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:49,751 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:49,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:49,751 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:49,752 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93118 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:49,752 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:49,752 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,752 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:49,753 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=90.9 K 2024-12-02T13:17:49,753 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:49,753 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,753 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,753 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5] 2024-12-02T13:17:49,753 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/2c4ffdee6d7f4f2d9972f2f277296fdc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.4 K 2024-12-02T13:17:49,753 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b07b3cb40049fc8333ec60eae614be, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:49,753 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4ffdee6d7f4f2d9972f2f277296fdc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:49,754 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9064544987734d6988c2dd7ab3eb8785, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733145467465 2024-12-02T13:17:49,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-02T13:17:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,755 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=161.02 KB heapSize=441.23 KB 2024-12-02T13:17:49,755 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37fbfa9ed5a4490590fe9554e8cb1cc5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:49,755 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 570ea49f38de4e318690b156595c20dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733145467465 2024-12-02T13:17:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:49,758 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ae809c4d92402f8926d0310671aa4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:49,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:49,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:49,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:49,782 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:49,791 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:49,792 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/233b8195ebd346f2af0c0da4601ebf66 is 50, key is test_row_0/B:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024b6af0340b10465781f862904e2cead6_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145469191/Put/seqid=0 2024-12-02T13:17:49,802 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412022fdcb28d28e444dcaefc1503a58bcde3_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:49,804 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412022fdcb28d28e444dcaefc1503a58bcde3_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:49,804 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022fdcb28d28e444dcaefc1503a58bcde3_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:49,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741965_1141 (size=12304) 2024-12-02T13:17:49,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741964_1140 (size=4469) 2024-12-02T13:17:49,836 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#117 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:49,837 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d0e65e6992684b9ca15e4c348912c894 is 175, key is test_row_0/A:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:49,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:49,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741966_1142 (size=31311) 2024-12-02T13:17:49,856 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d0e65e6992684b9ca15e4c348912c894 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894 2024-12-02T13:17:49,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145529852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145529854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145529854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145529855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,868 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into d0e65e6992684b9ca15e4c348912c894(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:49,868 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:49,868 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=13, startTime=1733145469750; duration=0sec 2024-12-02T13:17:49,868 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:49,868 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:49,868 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:49,871 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:49,871 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:49,872 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:49,872 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ea57b015324946adbf6f4dc88c1c237b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.4 K 2024-12-02T13:17:49,873 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea57b015324946adbf6f4dc88c1c237b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733145467300 2024-12-02T13:17:49,874 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 608cafcdb68c4ef1813bdf163ba09ed1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733145467465 2024-12-02T13:17:49,874 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a6adb332be44aecbe811ed7344d38dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:49,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741963_1139 (size=12357) 2024-12-02T13:17:49,891 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/233b8195ebd346f2af0c0da4601ebf66 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/233b8195ebd346f2af0c0da4601ebf66 2024-12-02T13:17:49,899 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into 233b8195ebd346f2af0c0da4601ebf66(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:49,899 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:49,899 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=13, startTime=1733145469751; duration=0sec 2024-12-02T13:17:49,899 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:49,899 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:49,903 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:49,907 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/3cb8507e32cf43039d2622f3a0eb5284 is 50, key is test_row_0/C:col10/1733145469099/Put/seqid=0 2024-12-02T13:17:49,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741967_1143 (size=12357) 2024-12-02T13:17:49,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145529964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145529964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:49,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:50,243 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024b6af0340b10465781f862904e2cead6_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024b6af0340b10465781f862904e2cead6_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/0fee35964d504510987a5cd95442d6bb, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/0fee35964d504510987a5cd95442d6bb is 175, key is test_row_0/A:col10/1733145469191/Put/seqid=0 2024-12-02T13:17:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:50,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741968_1144 (size=31105) 2024-12-02T13:17:50,341 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/3cb8507e32cf43039d2622f3a0eb5284 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3cb8507e32cf43039d2622f3a0eb5284 2024-12-02T13:17:50,348 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into 3cb8507e32cf43039d2622f3a0eb5284(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:50,348 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:50,348 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=13, startTime=1733145469758; duration=0sec 2024-12-02T13:17:50,348 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:50,348 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:50,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145530374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145530374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145530484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145530484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:50,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:50,690 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/0fee35964d504510987a5cd95442d6bb 2024-12-02T13:17:50,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a10aebe237ab441cbc534d39a19b3a5d is 50, key is test_row_0/B:col10/1733145469191/Put/seqid=0 2024-12-02T13:17:50,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741969_1145 (size=12151) 2024-12-02T13:17:50,755 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a10aebe237ab441cbc534d39a19b3a5d 2024-12-02T13:17:50,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1e762e15bb7346dea17cb56284e26342 is 50, key is test_row_0/C:col10/1733145469191/Put/seqid=0 2024-12-02T13:17:50,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741970_1146 (size=12151) 2024-12-02T13:17:50,832 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1e762e15bb7346dea17cb56284e26342 2024-12-02T13:17:50,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/0fee35964d504510987a5cd95442d6bb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb 2024-12-02T13:17:50,847 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb, entries=150, sequenceid=232, filesize=30.4 K 2024-12-02T13:17:50,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a10aebe237ab441cbc534d39a19b3a5d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d 2024-12-02T13:17:50,854 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d, entries=150, sequenceid=232, filesize=11.9 K 2024-12-02T13:17:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/1e762e15bb7346dea17cb56284e26342 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342 2024-12-02T13:17:50,871 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342, entries=150, sequenceid=232, filesize=11.9 K 2024-12-02T13:17:50,873 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~441.19 KB/451776, currentSize=26.84 KB/27480 for f8f6542858c1817d9eb99a42a221bf04 in 1117ms, sequenceid=232, compaction requested=false 2024-12-02T13:17:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:50,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-02T13:17:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-02T13:17:50,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-02T13:17:50,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-12-02T13:17:50,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7440 sec 2024-12-02T13:17:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:51,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:51,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:51,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120240b326830840433c8f4cad349a13bccb_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741971_1147 (size=14794) 2024-12-02T13:17:51,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:51,051 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120240b326830840433c8f4cad349a13bccb_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240b326830840433c8f4cad349a13bccb_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:51,052 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/59e70df6727e4d858df9a9ce61b75f90, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/59e70df6727e4d858df9a9ce61b75f90 is 175, key is test_row_0/A:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145471041, max timestamp=1733145471068Num uniques 859; 2024-12-02T13:17:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145471041, max timestamp=1733145471072Num uniques 859; 2024-12-02T13:17:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:51,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:51,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145471041, max timestamp=1733145471068Num uniques 859; 2024-12-02T13:17:51,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:17:51,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741972_1148 (size=39749) 2024-12-02T13:17:51,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/59e70df6727e4d858df9a9ce61b75f90 2024-12-02T13:17:51,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e395714578cd4be28d429d91bb89ce8f is 50, key is test_row_0/B:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741973_1149 (size=12151) 2024-12-02T13:17:51,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e395714578cd4be28d429d91bb89ce8f 2024-12-02T13:17:51,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f8dc165b129b47ec8d7995678e63dff6 is 50, key is test_row_0/C:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145531156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145531157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741974_1150 (size=12151) 2024-12-02T13:17:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145531266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145531284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-02T13:17:51,294 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:51,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T13:17:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T13:17:51,298 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:51,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:51,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:51,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145531384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145531384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,386 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,386 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T13:17:51,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T13:17:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:51,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145531474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145531494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f8dc165b129b47ec8d7995678e63dff6 2024-12-02T13:17:51,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/59e70df6727e4d858df9a9ce61b75f90 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90 2024-12-02T13:17:51,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90, entries=200, sequenceid=246, filesize=38.8 K 2024-12-02T13:17:51,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e395714578cd4be28d429d91bb89ce8f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f 2024-12-02T13:17:51,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f, entries=150, sequenceid=246, filesize=11.9 K 2024-12-02T13:17:51,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T13:17:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:51,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f8dc165b129b47ec8d7995678e63dff6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6 2024-12-02T13:17:51,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T13:17:51,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6, entries=150, sequenceid=246, filesize=11.9 K 2024-12-02T13:17:51,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f8f6542858c1817d9eb99a42a221bf04 in 610ms, sequenceid=246, compaction requested=true 2024-12-02T13:17:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:51,620 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:51,620 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:51,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:51,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:51,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:51,621 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:51,622 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:51,622 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,622 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=99.8 K 2024-12-02T13:17:51,622 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,622 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90] 2024-12-02T13:17:51,622 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:51,622 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e65e6992684b9ca15e4c348912c894, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:51,622 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:51,622 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,623 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/233b8195ebd346f2af0c0da4601ebf66, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.8 K 2024-12-02T13:17:51,624 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fee35964d504510987a5cd95442d6bb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733145469190 2024-12-02T13:17:51,624 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 233b8195ebd346f2af0c0da4601ebf66, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:51,624 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a10aebe237ab441cbc534d39a19b3a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733145469190 2024-12-02T13:17:51,624 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e70df6727e4d858df9a9ce61b75f90, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:51,624 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e395714578cd4be28d429d91bb89ce8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:51,638 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,639 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:51,641 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e50cbb77a8e74c9990293ed1153dec4d is 50, key is test_row_0/B:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,646 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202adfde7f3ad9c46a4a424b5c77d4d65b6_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202adfde7f3ad9c46a4a424b5c77d4d65b6_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202adfde7f3ad9c46a4a424b5c77d4d65b6_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741975_1151 (size=12459) 2024-12-02T13:17:51,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741976_1152 (size=4469) 2024-12-02T13:17:51,701 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#127 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:51,702 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/66eda8fe1e824e70949800f1b06cf3f7 is 175, key is test_row_0/A:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741977_1153 (size=31413) 2024-12-02T13:17:51,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:51,759 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:51,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120285282118ce5f4ecd8f183340a69b6f56_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145471141/Put/seqid=0 2024-12-02T13:17:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145531809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145531811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741978_1154 (size=12454) 2024-12-02T13:17:51,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:51,833 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120285282118ce5f4ecd8f183340a69b6f56_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120285282118ce5f4ecd8f183340a69b6f56_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/b3b7043d4b684ad1bb8933991494fd23, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/b3b7043d4b684ad1bb8933991494fd23 is 175, key is test_row_0/A:col10/1733145471141/Put/seqid=0 2024-12-02T13:17:51,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741979_1155 (size=31255) 2024-12-02T13:17:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/b3b7043d4b684ad1bb8933991494fd23 2024-12-02T13:17:51,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/54b6c141038d45b787b0cc144d0b0497 is 50, key is test_row_0/B:col10/1733145471141/Put/seqid=0 2024-12-02T13:17:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T13:17:51,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741980_1156 (size=12301) 2024-12-02T13:17:51,927 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/54b6c141038d45b787b0cc144d0b0497 2024-12-02T13:17:51,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145531923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145531924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:51,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/cc8c4d26914d499baa43f8eb44ab2d25 is 50, key is test_row_0/C:col10/1733145471141/Put/seqid=0 2024-12-02T13:17:51,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741981_1157 (size=12301) 2024-12-02T13:17:52,089 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e50cbb77a8e74c9990293ed1153dec4d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e50cbb77a8e74c9990293ed1153dec4d 2024-12-02T13:17:52,100 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into e50cbb77a8e74c9990293ed1153dec4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,101 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,101 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=13, startTime=1733145471620; duration=0sec 2024-12-02T13:17:52,101 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:52,101 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:52,101 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:52,102 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:52,102 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:52,102 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,102 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3cb8507e32cf43039d2622f3a0eb5284, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=35.8 K 2024-12-02T13:17:52,103 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cb8507e32cf43039d2622f3a0eb5284, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733145468461 2024-12-02T13:17:52,103 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e762e15bb7346dea17cb56284e26342, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733145469190 2024-12-02T13:17:52,104 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f8dc165b129b47ec8d7995678e63dff6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:52,115 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:52,116 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/bb41c0d8578f434a8fc130146704f906 is 50, key is test_row_0/C:col10/1733145469847/Put/seqid=0 2024-12-02T13:17:52,127 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/66eda8fe1e824e70949800f1b06cf3f7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7 2024-12-02T13:17:52,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741982_1158 (size=12459) 2024-12-02T13:17:52,143 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into 66eda8fe1e824e70949800f1b06cf3f7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,143 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,143 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=13, startTime=1733145471620; duration=0sec 2024-12-02T13:17:52,143 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,143 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145532134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145532144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,150 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/bb41c0d8578f434a8fc130146704f906 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/bb41c0d8578f434a8fc130146704f906 2024-12-02T13:17:52,158 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into bb41c0d8578f434a8fc130146704f906(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,158 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,158 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=13, startTime=1733145471621; duration=0sec 2024-12-02T13:17:52,158 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,158 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:52,268 INFO [master/06e3e18f2515:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T13:17:52,268 INFO [master/06e3e18f2515:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T13:17:52,365 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/cc8c4d26914d499baa43f8eb44ab2d25 2024-12-02T13:17:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/b3b7043d4b684ad1bb8933991494fd23 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23 2024-12-02T13:17:52,389 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23, entries=150, sequenceid=283, filesize=30.5 K 2024-12-02T13:17:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/54b6c141038d45b787b0cc144d0b0497 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497 2024-12-02T13:17:52,397 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T13:17:52,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/cc8c4d26914d499baa43f8eb44ab2d25 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25 2024-12-02T13:17:52,407 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T13:17:52,409 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f8f6542858c1817d9eb99a42a221bf04 in 650ms, sequenceid=283, compaction requested=false 2024-12-02T13:17:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T13:17:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T13:17:52,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T13:17:52,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-12-02T13:17:52,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.1190 sec 2024-12-02T13:17:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T13:17:52,434 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:52,435 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T13:17:52,437 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:52,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:52,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T13:17:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:52,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:52,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f5cb292a1d64ae8b4283c39b667e58b_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741983_1159 (size=12454) 2024-12-02T13:17:52,520 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145532518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f5cb292a1d64ae8b4283c39b667e58b_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f5cb292a1d64ae8b4283c39b667e58b_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145532521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,528 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3b9e93b0dc96497cbc64e39e8485392f, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3b9e93b0dc96497cbc64e39e8485392f is 175, key is test_row_0/A:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741984_1160 (size=31255) 2024-12-02T13:17:52,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3b9e93b0dc96497cbc64e39e8485392f 2024-12-02T13:17:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T13:17:52,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/30adc20a10f04dfc97e22c165ff1ad12 is 50, key is test_row_0/B:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741985_1161 (size=12301) 2024-12-02T13:17:52,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/30adc20a10f04dfc97e22c165ff1ad12 2024-12-02T13:17:52,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/25dbece239f848388295c83d3b096be6 is 50, key is test_row_0/C:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741986_1162 (size=12301) 2024-12-02T13:17:52,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T13:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:52,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:52,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:52,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/25dbece239f848388295c83d3b096be6 2024-12-02T13:17:52,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/3b9e93b0dc96497cbc64e39e8485392f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f 2024-12-02T13:17:52,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f, entries=150, sequenceid=298, filesize=30.5 K 2024-12-02T13:17:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/30adc20a10f04dfc97e22c165ff1ad12 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12 2024-12-02T13:17:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12, entries=150, sequenceid=298, filesize=12.0 K 2024-12-02T13:17:52,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/25dbece239f848388295c83d3b096be6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6 2024-12-02T13:17:52,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145532634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145532635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6, entries=150, sequenceid=298, filesize=12.0 K 2024-12-02T13:17:52,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f8f6542858c1817d9eb99a42a221bf04 in 191ms, sequenceid=298, compaction requested=true 2024-12-02T13:17:52,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:52,652 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:52,652 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:52,652 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,652 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=91.7 K 2024-12-02T13:17:52,653 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,653 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f] 2024-12-02T13:17:52,653 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66eda8fe1e824e70949800f1b06cf3f7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b7043d4b684ad1bb8933991494fd23, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733145471136 2024-12-02T13:17:52,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b9e93b0dc96497cbc64e39e8485392f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:52,656 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:52,657 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:52,657 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:52,657 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,658 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e50cbb77a8e74c9990293ed1153dec4d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=36.2 K 2024-12-02T13:17:52,658 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e50cbb77a8e74c9990293ed1153dec4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:52,659 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b6c141038d45b787b0cc144d0b0497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733145471136 2024-12-02T13:17:52,660 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 30adc20a10f04dfc97e22c165ff1ad12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:52,670 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412023cdaf9886eed43609d32a9c38c1dd9d6_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,683 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:52,683 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412023cdaf9886eed43609d32a9c38c1dd9d6_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,684 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a44c5fbee47e484bb664d0012a48905d is 50, key is test_row_0/B:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,684 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023cdaf9886eed43609d32a9c38c1dd9d6_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741987_1163 (size=4469) 2024-12-02T13:17:52,725 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#135 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:52,726 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/ce5a4ef226be44a3ab10fca8b7b5b3cc is 175, key is test_row_0/A:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741988_1164 (size=12711) 2024-12-02T13:17:52,744 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a44c5fbee47e484bb664d0012a48905d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a44c5fbee47e484bb664d0012a48905d 2024-12-02T13:17:52,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T13:17:52,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,752 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T13:17:52,755 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into a44c5fbee47e484bb664d0012a48905d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,755 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,755 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=13, startTime=1733145472656; duration=0sec 2024-12-02T13:17:52,755 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:52,755 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:52,755 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:52,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741989_1165 (size=31665) 2024-12-02T13:17:52,760 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:52,760 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:52,760 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:52,760 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/bb41c0d8578f434a8fc130146704f906, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=36.2 K 2024-12-02T13:17:52,761 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bb41c0d8578f434a8fc130146704f906, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145469847 2024-12-02T13:17:52,765 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8c4d26914d499baa43f8eb44ab2d25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733145471136 2024-12-02T13:17:52,766 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 25dbece239f848388295c83d3b096be6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:52,770 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/ce5a4ef226be44a3ab10fca8b7b5b3cc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc 2024-12-02T13:17:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d2646b54a4254ead836633cad49c91ce_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145472505/Put/seqid=0 2024-12-02T13:17:52,782 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into ce5a4ef226be44a3ab10fca8b7b5b3cc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,782 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,782 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=13, startTime=1733145472648; duration=0sec 2024-12-02T13:17:52,782 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,783 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:52,796 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:52,796 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/230cd8af0cd741f683647b61357b38be is 50, key is test_row_0/C:col10/1733145471809/Put/seqid=0 2024-12-02T13:17:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741990_1166 (size=12454) 2024-12-02T13:17:52,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:52,827 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d2646b54a4254ead836633cad49c91ce_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d2646b54a4254ead836633cad49c91ce_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c81606aed8bd48078a99fe755421d8a8, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:52,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c81606aed8bd48078a99fe755421d8a8 is 175, key is test_row_0/A:col10/1733145472505/Put/seqid=0 2024-12-02T13:17:52,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:52,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:52,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741991_1167 (size=12711) 2024-12-02T13:17:52,856 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/230cd8af0cd741f683647b61357b38be as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/230cd8af0cd741f683647b61357b38be 2024-12-02T13:17:52,871 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into 230cd8af0cd741f683647b61357b38be(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:52,871 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:52,871 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=13, startTime=1733145472668; duration=0sec 2024-12-02T13:17:52,871 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:52,871 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145532882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145532881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741992_1168 (size=31255) 2024-12-02T13:17:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145532994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145532994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T13:17:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145533203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145533204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,296 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c81606aed8bd48078a99fe755421d8a8 2024-12-02T13:17:53,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/05b389d916cd4643982673a436e3b063 is 50, key is test_row_0/B:col10/1733145472505/Put/seqid=0 2024-12-02T13:17:53,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741993_1169 (size=12301) 2024-12-02T13:17:53,340 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/05b389d916cd4643982673a436e3b063 2024-12-02T13:17:53,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c is 50, key is test_row_0/C:col10/1733145472505/Put/seqid=0 2024-12-02T13:17:53,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145533354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,357 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741994_1170 (size=12301) 2024-12-02T13:17:53,399 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c 2024-12-02T13:17:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145533404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,405 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c81606aed8bd48078a99fe755421d8a8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8 2024-12-02T13:17:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145533414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,415 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,416 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8, entries=150, sequenceid=318, filesize=30.5 K 2024-12-02T13:17:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/05b389d916cd4643982673a436e3b063 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063 2024-12-02T13:17:53,430 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063, entries=150, sequenceid=318, filesize=12.0 K 2024-12-02T13:17:53,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c 2024-12-02T13:17:53,438 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c, entries=150, sequenceid=318, filesize=12.0 K 2024-12-02T13:17:53,443 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f8f6542858c1817d9eb99a42a221bf04 in 691ms, sequenceid=318, compaction requested=false 2024-12-02T13:17:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T13:17:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T13:17:53,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T13:17:53,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-12-02T13:17:53,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0120 sec 2024-12-02T13:17:53,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:17:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:53,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f3eeb48b4d54433591741e3bc41fc683_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T13:17:53,574 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:53,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T13:17:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145533571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741995_1171 (size=17534) 2024-12-02T13:17:53,579 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:53,581 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:53,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145533577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f3eeb48b4d54433591741e3bc41fc683_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f3eeb48b4d54433591741e3bc41fc683_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,587 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c005444843724c418592e18e33fd1097, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:53,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c005444843724c418592e18e33fd1097 is 175, key is test_row_0/A:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:53,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741996_1172 (size=48639) 2024-12-02T13:17:53,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c005444843724c418592e18e33fd1097 2024-12-02T13:17:53,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/b30476e01da94e24b354ef49fd6f820c is 50, key is test_row_0/B:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741997_1173 (size=12301) 2024-12-02T13:17:53,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/b30476e01da94e24b354ef49fd6f820c 2024-12-02T13:17:53,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145533684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/ee24397bdb514ce384793a47ab53b543 is 50, key is test_row_0/C:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145533695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741998_1174 (size=12301) 2024-12-02T13:17:53,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T13:17:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:53,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:53,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:53,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T13:17:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:53,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:53,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145533894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:53,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:53,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145533904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:53,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T13:17:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:54,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/ee24397bdb514ce384793a47ab53b543 2024-12-02T13:17:54,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/c005444843724c418592e18e33fd1097 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097 2024-12-02T13:17:54,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097, entries=250, sequenceid=335, filesize=47.5 K 2024-12-02T13:17:54,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/b30476e01da94e24b354ef49fd6f820c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c 2024-12-02T13:17:54,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T13:17:54,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/ee24397bdb514ce384793a47ab53b543 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543 2024-12-02T13:17:54,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T13:17:54,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f8f6542858c1817d9eb99a42a221bf04 in 650ms, sequenceid=335, compaction requested=true 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:54,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:54,167 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:54,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:54,172 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:54,173 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111559 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:54,173 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:54,173 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,173 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=108.9 K 2024-12-02T13:17:54,173 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,174 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097] 2024-12-02T13:17:54,174 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:54,174 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,174 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a44c5fbee47e484bb664d0012a48905d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=36.4 K 2024-12-02T13:17:54,174 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5a4ef226be44a3ab10fca8b7b5b3cc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:54,174 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a44c5fbee47e484bb664d0012a48905d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:54,175 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c81606aed8bd48078a99fe755421d8a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733145472505 2024-12-02T13:17:54,175 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b389d916cd4643982673a436e3b063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733145472505 2024-12-02T13:17:54,175 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c005444843724c418592e18e33fd1097, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472872 2024-12-02T13:17:54,176 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b30476e01da94e24b354ef49fd6f820c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472874 2024-12-02T13:17:54,198 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:54,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T13:17:54,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,200 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:54,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:54,204 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:54,205 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a0f4f7c679164e62b1f56220761d45ec is 50, key is test_row_0/B:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:54,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:54,213 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412022f701a60f4414537a37192acce2703f3_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:54,215 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412022f701a60f4414537a37192acce2703f3_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:54,216 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f701a60f4414537a37192acce2703f3_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022320e59d7b1f4c30af06a38e0eb8f645_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145473541/Put/seqid=0 2024-12-02T13:17:54,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741999_1175 (size=12813) 2024-12-02T13:17:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145534242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145534249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,258 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a0f4f7c679164e62b1f56220761d45ec as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a0f4f7c679164e62b1f56220761d45ec 2024-12-02T13:17:54,267 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into a0f4f7c679164e62b1f56220761d45ec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:54,268 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:54,268 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=13, startTime=1733145474167; duration=0sec 2024-12-02T13:17:54,268 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:54,269 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:54,269 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:17:54,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742000_1176 (size=4469) 2024-12-02T13:17:54,273 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:17:54,274 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#144 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:54,274 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:54,274 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:54,274 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/230cd8af0cd741f683647b61357b38be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=36.4 K 2024-12-02T13:17:54,274 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/66038be181294f99b4e9969d41b381db is 175, key is test_row_0/A:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:54,275 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 230cd8af0cd741f683647b61357b38be, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733145471809 2024-12-02T13:17:54,275 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ddacce6a69f4f3a92a5d8e6c8bbe57c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733145472505 2024-12-02T13:17:54,275 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ee24397bdb514ce384793a47ab53b543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472874 2024-12-02T13:17:54,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742001_1177 (size=12454) 2024-12-02T13:17:54,304 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:54,305 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2fe2061028b7440e90d956fc9c5a8ff4 is 50, key is test_row_0/C:col10/1733145473516/Put/seqid=0 2024-12-02T13:17:54,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742002_1178 (size=31767) 2024-12-02T13:17:54,324 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/66038be181294f99b4e9969d41b381db as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db 2024-12-02T13:17:54,331 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into 66038be181294f99b4e9969d41b381db(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:54,331 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:54,331 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=13, startTime=1733145474167; duration=0sec 2024-12-02T13:17:54,332 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:54,332 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742003_1179 (size=12813) 2024-12-02T13:17:54,362 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2fe2061028b7440e90d956fc9c5a8ff4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2fe2061028b7440e90d956fc9c5a8ff4 2024-12-02T13:17:54,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145534364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145534364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,371 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into 2fe2061028b7440e90d956fc9c5a8ff4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:54,371 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:54,372 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=13, startTime=1733145474167; duration=0sec 2024-12-02T13:17:54,372 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:54,372 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145534574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145534574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:54,694 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022320e59d7b1f4c30af06a38e0eb8f645_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022320e59d7b1f4c30af06a38e0eb8f645_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/96d38517f4f542aa8c59fb5dc0e7a314, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/96d38517f4f542aa8c59fb5dc0e7a314 is 175, key is test_row_0/A:col10/1733145473541/Put/seqid=0 2024-12-02T13:17:54,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742004_1180 (size=31255) 2024-12-02T13:17:54,710 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/96d38517f4f542aa8c59fb5dc0e7a314 2024-12-02T13:17:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/899e97234e674c8e898c2876cfae217e is 50, key is test_row_0/B:col10/1733145473541/Put/seqid=0 2024-12-02T13:17:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:54,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742005_1181 (size=12301) 2024-12-02T13:17:54,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145534893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145534893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:54,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:55,129 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/899e97234e674c8e898c2876cfae217e 2024-12-02T13:17:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2733e54761b54b7593387a2d55d75fe3 is 50, key is test_row_0/C:col10/1733145473541/Put/seqid=0 2024-12-02T13:17:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742006_1182 (size=12301) 2024-12-02T13:17:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145535404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145535404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:55,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:55,557 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2733e54761b54b7593387a2d55d75fe3 2024-12-02T13:17:55,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/96d38517f4f542aa8c59fb5dc0e7a314 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314 2024-12-02T13:17:55,577 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314, entries=150, sequenceid=353, filesize=30.5 K 2024-12-02T13:17:55,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/899e97234e674c8e898c2876cfae217e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e 2024-12-02T13:17:55,584 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e, entries=150, sequenceid=353, filesize=12.0 K 2024-12-02T13:17:55,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2733e54761b54b7593387a2d55d75fe3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3 2024-12-02T13:17:55,592 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3, entries=150, sequenceid=353, filesize=12.0 K 2024-12-02T13:17:55,593 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f8f6542858c1817d9eb99a42a221bf04 in 1393ms, sequenceid=353, compaction requested=false 2024-12-02T13:17:55,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:55,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:55,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T13:17:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T13:17:55,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T13:17:55,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-12-02T13:17:55,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0220 sec 2024-12-02T13:17:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T13:17:55,734 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:55,734 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T13:17:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T13:17:55,738 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:55,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:55,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T13:17:55,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T13:17:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:55,894 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba2c11c10c454102be5b2b9e23498148_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145474240/Put/seqid=0 2024-12-02T13:17:55,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742007_1183 (size=12454) 2024-12-02T13:17:55,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:55,966 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba2c11c10c454102be5b2b9e23498148_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba2c11c10c454102be5b2b9e23498148_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:55,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/591b4f92bab346fea7394d4ee230f844, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/591b4f92bab346fea7394d4ee230f844 is 175, key is test_row_0/A:col10/1733145474240/Put/seqid=0 2024-12-02T13:17:55,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742008_1184 (size=31255) 2024-12-02T13:17:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/591b4f92bab346fea7394d4ee230f844 2024-12-02T13:17:55,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c7e8fe478aff42ed874dd943631024a9 is 50, key is test_row_0/B:col10/1733145474240/Put/seqid=0 2024-12-02T13:17:56,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742009_1185 (size=12301) 2024-12-02T13:17:56,024 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c7e8fe478aff42ed874dd943631024a9 2024-12-02T13:17:56,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b6864d3da6d740c799cd6d4473d4f2f5 is 50, key is test_row_0/C:col10/1733145474240/Put/seqid=0 2024-12-02T13:17:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T13:17:56,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742010_1186 (size=12301) 2024-12-02T13:17:56,099 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b6864d3da6d740c799cd6d4473d4f2f5 2024-12-02T13:17:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/591b4f92bab346fea7394d4ee230f844 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844 2024-12-02T13:17:56,110 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844, entries=150, sequenceid=370, filesize=30.5 K 2024-12-02T13:17:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c7e8fe478aff42ed874dd943631024a9 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9 2024-12-02T13:17:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,117 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9, entries=150, sequenceid=370, filesize=12.0 K 2024-12-02T13:17:56,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b6864d3da6d740c799cd6d4473d4f2f5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5 2024-12-02T13:17:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,134 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5, entries=150, sequenceid=370, filesize=12.0 K 2024-12-02T13:17:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for f8f6542858c1817d9eb99a42a221bf04 in 241ms, sequenceid=370, compaction requested=true 2024-12-02T13:17:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T13:17:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T13:17:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T13:17:56,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 398 msec 2024-12-02T13:17:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 406 msec 2024-12-02T13:17:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T13:17:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,365 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:56,365 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:56,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T13:17:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:56,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,372 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,373 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T13:17:56,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:56,527 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T13:17:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:56,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:17:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b7c2e4d838374389ad82386e808f1e67_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145476526/Put/seqid=0 2024-12-02T13:17:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742011_1187 (size=17534) 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145476623, max timestamp=1733145476635Num uniques 859; 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145476623, max timestamp=1733145476635Num uniques 859; 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145476623, max timestamp=1733145476635Num uniques 859; 2024-12-02T13:17:56,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:56,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145536720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:56,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145536721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145536848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:56,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145536848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:56,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:56,990 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b7c2e4d838374389ad82386e808f1e67_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b7c2e4d838374389ad82386e808f1e67_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c2b28a7924d4e65afe7946129bbb26c, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:56,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c2b28a7924d4e65afe7946129bbb26c is 175, key is test_row_0/A:col10/1733145476526/Put/seqid=0 2024-12-02T13:17:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:57,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742012_1188 (size=48635) 2024-12-02T13:17:57,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145537064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145537064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145537374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145537374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,431 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c2b28a7924d4e65afe7946129bbb26c 2024-12-02T13:17:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50514 deadline: 1733145537434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,435 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e8d062bb4d6544d48542f6162fc7a6bc is 50, key is test_row_0/B:col10/1733145476526/Put/seqid=0 2024-12-02T13:17:57,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742013_1189 (size=9857) 2024-12-02T13:17:57,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50556 deadline: 1733145537454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:57,459 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:17:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:57,459 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e8d062bb4d6544d48542f6162fc7a6bc 2024-12-02T13:17:57,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/a3d188b401504f5d9ff32ccf6d52e600 is 50, key is test_row_0/C:col10/1733145476526/Put/seqid=0 2024-12-02T13:17:57,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742014_1190 (size=9857) 2024-12-02T13:17:57,479 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/a3d188b401504f5d9ff32ccf6d52e600 2024-12-02T13:17:57,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c2b28a7924d4e65afe7946129bbb26c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c 2024-12-02T13:17:57,501 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c, entries=250, sequenceid=379, filesize=47.5 K 2024-12-02T13:17:57,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/e8d062bb4d6544d48542f6162fc7a6bc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc 2024-12-02T13:17:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc, entries=100, sequenceid=379, filesize=9.6 K 2024-12-02T13:17:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/a3d188b401504f5d9ff32ccf6d52e600 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600 2024-12-02T13:17:57,523 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600, entries=100, sequenceid=379, filesize=9.6 K 2024-12-02T13:17:57,524 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f8f6542858c1817d9eb99a42a221bf04 in 997ms, sequenceid=379, compaction requested=true 2024-12-02T13:17:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T13:17:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T13:17:57,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T13:17:57,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-12-02T13:17:57,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1630 sec 2024-12-02T13:17:57,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145476704, max timestamp=1733145477888Num uniques 1068; 2024-12-02T13:17:57,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145476704, max timestamp=1733145477888Num uniques 1068; 2024-12-02T13:17:57,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:17:57,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:17:57,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=73.80 KB heapSize=203.80 KB 2024-12-02T13:17:57,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:57,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145476704, max timestamp=1733145477888Num uniques 1068; 2024-12-02T13:17:57,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:57,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dcf07d0bad6040aa832dd01bfc553d16_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742015_1191 (size=12454) 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733145477946, max timestamp=1733145477980Num uniques 859; 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733145477946, max timestamp=1733145477980Num uniques 859; 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733145477946, max timestamp=1733145477980Num uniques 859; 2024-12-02T13:17:57,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-02T13:17:58,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733145478033, max timestamp=1733145478049Num uniques 1218; 2024-12-02T13:17:58,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733145478033, max timestamp=1733145478049Num uniques 1218; 2024-12-02T13:17:58,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=9978, suffix heap size=154640,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=218 2024-12-02T13:17:58,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=9978, suffix heap size=154640,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=218 2024-12-02T13:17:58,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733145478033, max timestamp=1733145478049Num uniques 1218; 2024-12-02T13:17:58,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=9978, suffix heap size=154640,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=218 2024-12-02T13:17:58,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733145478096, max timestamp=1733145478124Num uniques 1077; 2024-12-02T13:17:58,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-02T13:17:58,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733145478112, max timestamp=1733145478124Num uniques 1077; 2024-12-02T13:17:58,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-12-02T13:17:58,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733145478096, max timestamp=1733145478124Num uniques 1077; 2024-12-02T13:17:58,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-02T13:17:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145478167, max timestamp=1733145478208Num uniques 1136; 2024-12-02T13:17:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T13:17:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145478167, max timestamp=1733145478208Num uniques 1136; 2024-12-02T13:17:58,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-02T13:17:58,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145478167, max timestamp=1733145478208Num uniques 1009; 2024-12-02T13:17:58,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-12-02T13:17:58,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733145478247, max timestamp=1733145478272Num uniques 1045; 2024-12-02T13:17:58,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-02T13:17:58,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733145478247, max timestamp=1733145478272Num uniques 1045; 2024-12-02T13:17:58,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-02T13:17:58,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733145478247, max timestamp=1733145478272Num uniques 1045; 2024-12-02T13:17:58,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-02T13:17:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733145478293, max timestamp=1733145478349Num uniques 1104; 2024-12-02T13:17:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733145478293, max timestamp=1733145478349Num uniques 1104; 2024-12-02T13:17:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-02T13:17:58,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-02T13:17:58,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733145478293, max timestamp=1733145478349Num uniques 1104; 2024-12-02T13:17:58,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-12-02T13:17:58,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:58,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dcf07d0bad6040aa832dd01bfc553d16_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dcf07d0bad6040aa832dd01bfc553d16_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:58,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/88ece19fc8cf46dcb180a1bfb19fc1d6, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/88ece19fc8cf46dcb180a1bfb19fc1d6 is 175, key is test_row_0/A:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742016_1192 (size=31255) 2024-12-02T13:17:58,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733145478412, max timestamp=1733145478429Num uniques 1113; 2024-12-02T13:17:58,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-02T13:17:58,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733145478412, max timestamp=1733145478429Num uniques 1113; 2024-12-02T13:17:58,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733145478412, max timestamp=1733145478429Num uniques 1113; 2024-12-02T13:17:58,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-02T13:17:58,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-02T13:17:58,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/88ece19fc8cf46dcb180a1bfb19fc1d6 2024-12-02T13:17:58,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a881fdfe305b4b0c8a5863cf32b11e9d is 50, key is test_row_0/B:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742017_1193 (size=12301) 2024-12-02T13:17:58,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a881fdfe305b4b0c8a5863cf32b11e9d 2024-12-02T13:17:58,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b8b73be2b9b94fe9b8b1ba316e5fa170 is 50, key is test_row_0/C:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733145478472, max timestamp=1733145478506Num uniques 1072; 2024-12-02T13:17:58,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-02T13:17:58,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733145478472, max timestamp=1733145478506Num uniques 1072; 2024-12-02T13:17:58,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1733145478472, max timestamp=1733145478506Num uniques 1072; 2024-12-02T13:17:58,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-02T13:17:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T13:17:58,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:17:58,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:17:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T13:17:58,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:17:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:17:58,517 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:17:58,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:17:58,517 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-12-02T13:17:58,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742018_1194 (size=12301) 2024-12-02T13:17:58,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b8b73be2b9b94fe9b8b1ba316e5fa170 2024-12-02T13:17:58,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/88ece19fc8cf46dcb180a1bfb19fc1d6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6 2024-12-02T13:17:58,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6, entries=150, sequenceid=418, filesize=30.5 K 2024-12-02T13:17:58,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a881fdfe305b4b0c8a5863cf32b11e9d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d 2024-12-02T13:17:58,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d, entries=150, sequenceid=418, filesize=12.0 K 2024-12-02T13:17:58,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/b8b73be2b9b94fe9b8b1ba316e5fa170 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170 2024-12-02T13:17:58,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733145478560, max timestamp=1733145478578Num uniques 1131; 2024-12-02T13:17:58,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733145478560, max timestamp=1733145478578Num uniques 1131; 2024-12-02T13:17:58,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-02T13:17:58,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-02T13:17:58,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170, entries=150, sequenceid=418, filesize=12.0 K 2024-12-02T13:17:58,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=73.80 KB/75570 for f8f6542858c1817d9eb99a42a221bf04 in 692ms, sequenceid=418, compaction requested=true 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:17:58,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733145478560, max timestamp=1733145478578Num uniques 1131; 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:58,586 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:17:58,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:17:58,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-12-02T13:17:58,587 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:17:58,589 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59573 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:17:58,590 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:17:58,590 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,590 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a0f4f7c679164e62b1f56220761d45ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=58.2 K 2024-12-02T13:17:58,590 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 174167 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:17:58,590 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:17:58,590 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,590 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=170.1 K 2024-12-02T13:17:58,590 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f4f7c679164e62b1f56220761d45ec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472874 2024-12-02T13:17:58,590 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,590 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6] 2024-12-02T13:17:58,591 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 899e97234e674c8e898c2876cfae217e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733145473541 2024-12-02T13:17:58,591 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 66038be181294f99b4e9969d41b381db, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472874 2024-12-02T13:17:58,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e8fe478aff42ed874dd943631024a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733145474220 2024-12-02T13:17:58,592 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d38517f4f542aa8c59fb5dc0e7a314, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733145473541 2024-12-02T13:17:58,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d062bb4d6544d48542f6162fc7a6bc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733145476472 2024-12-02T13:17:58,592 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 591b4f92bab346fea7394d4ee230f844, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733145474220 2024-12-02T13:17:58,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a881fdfe305b4b0c8a5863cf32b11e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:17:58,593 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c2b28a7924d4e65afe7946129bbb26c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733145476472 2024-12-02T13:17:58,593 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ece19fc8cf46dcb180a1bfb19fc1d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:17:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:58,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=53.67 KB heapSize=139.17 KB 2024-12-02T13:17:58,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:17:58,626 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:58,627 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/694e03ce75dd4d82acdf6519ecbada3a is 50, key is test_row_0/B:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,632 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120291b5025f68c64ec1b93d02271c5006d9_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145478596/Put/seqid=0 2024-12-02T13:17:58,638 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202be31766bb4174e16837f5142fabdb3ca_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,639 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202be31766bb4174e16837f5142fabdb3ca_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,640 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202be31766bb4174e16837f5142fabdb3ca_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742019_1195 (size=12983) 2024-12-02T13:17:58,665 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/694e03ce75dd4d82acdf6519ecbada3a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/694e03ce75dd4d82acdf6519ecbada3a 2024-12-02T13:17:58,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T13:17:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145478638, max timestamp=1733145478667Num uniques 859; 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145478638, max timestamp=1733145478667Num uniques 859; 2024-12-02T13:17:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145538668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145478638, max timestamp=1733145478667Num uniques 859; 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:58,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:17:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,674 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into 694e03ce75dd4d82acdf6519ecbada3a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:58,674 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:58,674 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=11, startTime=1733145478586; duration=0sec 2024-12-02T13:17:58,674 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:17:58,674 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:17:58,674 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:17:58,684 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59573 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:17:58,684 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:17:58,684 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,684 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2fe2061028b7440e90d956fc9c5a8ff4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=58.2 K 2024-12-02T13:17:58,684 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe2061028b7440e90d956fc9c5a8ff4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733145472874 2024-12-02T13:17:58,685 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2733e54761b54b7593387a2d55d75fe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733145473541 2024-12-02T13:17:58,685 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6864d3da6d740c799cd6d4473d4f2f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733145474220 2024-12-02T13:17:58,685 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d188b401504f5d9ff32ccf6d52e600, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733145476472 2024-12-02T13:17:58,686 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b73be2b9b94fe9b8b1ba316e5fa170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:17:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742020_1196 (size=17534) 2024-12-02T13:17:58,702 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:58,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120291b5025f68c64ec1b93d02271c5006d9_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120291b5025f68c64ec1b93d02271c5006d9_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:58,708 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/af3d5696be3c4bca979a73108e6030fd, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:58,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/af3d5696be3c4bca979a73108e6030fd is 175, key is test_row_0/A:col10/1733145478596/Put/seqid=0 2024-12-02T13:17:58,728 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:58,729 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/43f5981f28d04183be1a636e04073e26 is 50, key is test_row_0/C:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742021_1197 (size=4469) 2024-12-02T13:17:58,733 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#160 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:17:58,734 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/ea90ef56638847f2a5a70ccb8818b17b is 175, key is test_row_0/A:col10/1733145476704/Put/seqid=0 2024-12-02T13:17:58,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742023_1199 (size=12983) 2024-12-02T13:17:58,777 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/43f5981f28d04183be1a636e04073e26 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/43f5981f28d04183be1a636e04073e26 2024-12-02T13:17:58,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742024_1200 (size=31937) 2024-12-02T13:17:58,791 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into 43f5981f28d04183be1a636e04073e26(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:58,791 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:58,791 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=11, startTime=1733145478586; duration=0sec 2024-12-02T13:17:58,791 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:58,791 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:17:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145538789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:58,798 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/ea90ef56638847f2a5a70ccb8818b17b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b 2024-12-02T13:17:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145538792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:58,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742022_1198 (size=48639) 2024-12-02T13:17:58,805 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=580, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/af3d5696be3c4bca979a73108e6030fd 2024-12-02T13:17:58,806 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into ea90ef56638847f2a5a70ccb8818b17b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:17:58,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:58,806 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=11, startTime=1733145478586; duration=0sec 2024-12-02T13:17:58,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:17:58,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:17:58,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T13:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:17:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:17:58,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f4758ae1eb814bba9aabda5584b9b0ac is 50, key is test_row_0/B:col10/1733145478596/Put/seqid=0 2024-12-02T13:17:58,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742025_1201 (size=12301) 2024-12-02T13:17:58,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f4758ae1eb814bba9aabda5584b9b0ac 2024-12-02T13:17:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145538903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145538914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:58,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f30fed3e439c4b33b67b5b638eac4a6d is 50, key is test_row_0/C:col10/1733145478596/Put/seqid=0 2024-12-02T13:17:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:58,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:58,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742026_1202 (size=12301) 2024-12-02T13:17:58,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f30fed3e439c4b33b67b5b638eac4a6d 2024-12-02T13:17:58,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/af3d5696be3c4bca979a73108e6030fd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd 2024-12-02T13:17:58,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd, entries=250, sequenceid=580, filesize=47.5 K 2024-12-02T13:17:58,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f4758ae1eb814bba9aabda5584b9b0ac as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac 2024-12-02T13:17:58,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac, entries=150, sequenceid=580, filesize=12.0 K 2024-12-02T13:17:58,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/f30fed3e439c4b33b67b5b638eac4a6d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d 2024-12-02T13:17:58,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d, entries=150, sequenceid=580, filesize=12.0 K 2024-12-02T13:17:58,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~159.05 KB/162864, currentSize=134.18 KB/137400 for f8f6542858c1817d9eb99a42a221bf04 in 358ms, sequenceid=580, compaction requested=false 2024-12-02T13:17:58,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:58,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T13:17:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:58,984 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:17:59,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f9f1861ec2e4f2484dc437c659bddbd_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145478788/Put/seqid=0 2024-12-02T13:17:59,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742027_1203 (size=12454) 2024-12-02T13:17:59,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:17:59,024 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f9f1861ec2e4f2484dc437c659bddbd_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9f1861ec2e4f2484dc437c659bddbd_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:59,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c825dc7587141c49bce9882af0e3861, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:17:59,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c825dc7587141c49bce9882af0e3861 is 175, key is test_row_0/A:col10/1733145478788/Put/seqid=0 2024-12-02T13:17:59,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742028_1204 (size=31255) 2024-12-02T13:17:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:17:59,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:17:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145539129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145539131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:17:59,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145539234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145539244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,442 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=619, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c825dc7587141c49bce9882af0e3861 2024-12-02T13:17:59,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145539443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145539454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/335623f41965410396db34997c6625ad is 50, key is test_row_0/B:col10/1733145478788/Put/seqid=0 2024-12-02T13:17:59,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742029_1205 (size=12301) 2024-12-02T13:17:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:17:59,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145539754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:17:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145539764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:17:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:17:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:17:59,865 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/335623f41965410396db34997c6625ad 2024-12-02T13:17:59,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/daeed142fdb64776a383c282db436d6f is 50, key is test_row_0/C:col10/1733145478788/Put/seqid=0 2024-12-02T13:17:59,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742030_1206 (size=12301) 2024-12-02T13:17:59,881 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/daeed142fdb64776a383c282db436d6f 2024-12-02T13:17:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1c825dc7587141c49bce9882af0e3861 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861 2024-12-02T13:17:59,891 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861, entries=150, sequenceid=619, filesize=30.5 K 2024-12-02T13:17:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/335623f41965410396db34997c6625ad as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad 2024-12-02T13:17:59,897 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad, entries=150, sequenceid=619, filesize=12.0 K 2024-12-02T13:17:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/daeed142fdb64776a383c282db436d6f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f 2024-12-02T13:17:59,902 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f, entries=150, sequenceid=619, filesize=12.0 K 2024-12-02T13:17:59,903 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f8f6542858c1817d9eb99a42a221bf04 in 918ms, sequenceid=619, compaction requested=true 2024-12-02T13:17:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:17:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:17:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T13:17:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T13:17:59,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T13:17:59,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3870 sec 2024-12-02T13:17:59,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.3920 sec 2024-12-02T13:18:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:00,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:00,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:00,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ca004cf8c7364a788bd6011e1d7cacf3_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742031_1207 (size=14994) 2024-12-02T13:18:00,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145480284, max timestamp=1733145480306Num uniques 859; 2024-12-02T13:18:00,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:00,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145480284, max timestamp=1733145480306Num uniques 859; 2024-12-02T13:18:00,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:00,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145480284, max timestamp=1733145480306Num uniques 859; 2024-12-02T13:18:00,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:00,314 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ca004cf8c7364a788bd6011e1d7cacf3_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca004cf8c7364a788bd6011e1d7cacf3_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:00,315 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1aa8646b18e44762b6e698fcfe7a538d, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:00,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1aa8646b18e44762b6e698fcfe7a538d is 175, key is test_row_0/A:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742032_1208 (size=39949) 2024-12-02T13:18:00,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145540339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145540344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145540453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145540454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T13:18:00,664 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:00,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-02T13:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:00,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:00,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:00,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:00,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145540664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145540664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,720 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=630, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1aa8646b18e44762b6e698fcfe7a538d 2024-12-02T13:18:00,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f38c361affdd45bab005ef41e29aa563 is 50, key is test_row_0/B:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742033_1209 (size=12301) 2024-12-02T13:18:00,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f38c361affdd45bab005ef41e29aa563 2024-12-02T13:18:00,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2e8e8a151c8a4a2892b352a3e7fdb575 is 50, key is test_row_0/C:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742034_1210 (size=12301) 2024-12-02T13:18:00,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2e8e8a151c8a4a2892b352a3e7fdb575 2024-12-02T13:18:00,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/1aa8646b18e44762b6e698fcfe7a538d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d 2024-12-02T13:18:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:00,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d, entries=200, sequenceid=630, filesize=39.0 K 2024-12-02T13:18:00,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f38c361affdd45bab005ef41e29aa563 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563 2024-12-02T13:18:00,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563, entries=150, sequenceid=630, filesize=12.0 K 2024-12-02T13:18:00,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2e8e8a151c8a4a2892b352a3e7fdb575 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575 2024-12-02T13:18:00,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575, entries=150, sequenceid=630, filesize=12.0 K 2024-12-02T13:18:00,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f8f6542858c1817d9eb99a42a221bf04 in 527ms, sequenceid=630, compaction requested=true 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:00,793 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:00,793 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:00,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:00,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:00,795 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:00,795 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:18:00,795 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:00,795 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/694e03ce75dd4d82acdf6519ecbada3a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=48.7 K 2024-12-02T13:18:00,796 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:00,796 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:18:00,796 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 694e03ce75dd4d82acdf6519ecbada3a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:18:00,796 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:00,797 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=148.2 K 2024-12-02T13:18:00,797 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:00,797 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d] 2024-12-02T13:18:00,797 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f4758ae1eb814bba9aabda5584b9b0ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733145478567 2024-12-02T13:18:00,797 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 335623f41965410396db34997c6625ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733145478735 2024-12-02T13:18:00,797 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea90ef56638847f2a5a70ccb8818b17b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:18:00,797 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting af3d5696be3c4bca979a73108e6030fd, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733145478567 2024-12-02T13:18:00,798 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f38c361affdd45bab005ef41e29aa563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479126 2024-12-02T13:18:00,798 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c825dc7587141c49bce9882af0e3861, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733145478735 2024-12-02T13:18:00,798 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa8646b18e44762b6e698fcfe7a538d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479124 2024-12-02T13:18:00,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:00,819 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:00,831 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:00,831 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/af6a9fbbab5b483881b1186070d176be is 50, key is test_row_0/B:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,833 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:00,841 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412023f4edb4ef32548a9b7cf05f3d8837a16_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:00,843 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412023f4edb4ef32548a9b7cf05f3d8837a16_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:00,844 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023f4edb4ef32548a9b7cf05f3d8837a16_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:00,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024363a2bec5e648b6ab51a0287f8826d0_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145480336/Put/seqid=0 2024-12-02T13:18:00,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742035_1211 (size=13119) 2024-12-02T13:18:00,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742036_1212 (size=4469) 2024-12-02T13:18:00,862 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#172 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:00,862 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/caca07f53d2f4abd9cc73cd3269c185f is 175, key is test_row_0/A:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:00,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742037_1213 (size=12454) 2024-12-02T13:18:00,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742038_1214 (size=32073) 2024-12-02T13:18:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:00,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:00,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145540989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145540990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:00,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145541104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145541104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,258 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/af6a9fbbab5b483881b1186070d176be as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af6a9fbbab5b483881b1186070d176be 2024-12-02T13:18:01,263 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into af6a9fbbab5b483881b1186070d176be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:01,263 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:01,263 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=12, startTime=1733145480793; duration=0sec 2024-12-02T13:18:01,263 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:01,263 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:18:01,264 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:01,265 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:01,265 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:18:01,265 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:01,265 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/43f5981f28d04183be1a636e04073e26, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=48.7 K 2024-12-02T13:18:01,266 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f5981f28d04183be1a636e04073e26, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733145476704 2024-12-02T13:18:01,266 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f30fed3e439c4b33b67b5b638eac4a6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=580, earliestPutTs=1733145478567 2024-12-02T13:18:01,266 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting daeed142fdb64776a383c282db436d6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1733145478735 2024-12-02T13:18:01,266 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e8e8a151c8a4a2892b352a3e7fdb575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479126 2024-12-02T13:18:01,273 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:01,274 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/3871d0051aa74c7793a313c9119c58b6 is 50, key is test_row_0/C:col10/1733145479126/Put/seqid=0 2024-12-02T13:18:01,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:01,282 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024363a2bec5e648b6ab51a0287f8826d0_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024363a2bec5e648b6ab51a0287f8826d0_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/f1889d6093a74cc1a5c9960728c88497, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:01,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/f1889d6093a74cc1a5c9960728c88497 is 175, key is test_row_0/A:col10/1733145480336/Put/seqid=0 2024-12-02T13:18:01,290 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/caca07f53d2f4abd9cc73cd3269c185f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f 2024-12-02T13:18:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:01,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742039_1215 (size=13119) 2024-12-02T13:18:01,301 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into caca07f53d2f4abd9cc73cd3269c185f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:01,301 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:01,301 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=12, startTime=1733145480793; duration=0sec 2024-12-02T13:18:01,301 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:01,301 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:18:01,308 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/3871d0051aa74c7793a313c9119c58b6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3871d0051aa74c7793a313c9119c58b6 2024-12-02T13:18:01,315 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into 3871d0051aa74c7793a313c9119c58b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:01,315 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:01,315 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=12, startTime=1733145480793; duration=0sec 2024-12-02T13:18:01,316 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:01,316 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:18:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145541314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145541314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742040_1216 (size=31255) 2024-12-02T13:18:01,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145541623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145541623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:01,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:01,719 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=664, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/f1889d6093a74cc1a5c9960728c88497 2024-12-02T13:18:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eecccb1a634b4417b4d90c5ceacfba5f is 50, key is test_row_0/B:col10/1733145480336/Put/seqid=0 2024-12-02T13:18:01,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742041_1217 (size=12301) 2024-12-02T13:18:01,735 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eecccb1a634b4417b4d90c5ceacfba5f 2024-12-02T13:18:01,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/e412a6143e5d45c99eb4b4c26fadff1d is 50, key is test_row_0/C:col10/1733145480336/Put/seqid=0 2024-12-02T13:18:01,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742042_1218 (size=12301) 2024-12-02T13:18:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145542133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:02,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145542133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:02,171 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/e412a6143e5d45c99eb4b4c26fadff1d 2024-12-02T13:18:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/f1889d6093a74cc1a5c9960728c88497 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497 2024-12-02T13:18:02,179 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497, entries=150, sequenceid=664, filesize=30.5 K 2024-12-02T13:18:02,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eecccb1a634b4417b4d90c5ceacfba5f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f 2024-12-02T13:18:02,183 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f, entries=150, sequenceid=664, filesize=12.0 K 2024-12-02T13:18:02,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/e412a6143e5d45c99eb4b4c26fadff1d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d 2024-12-02T13:18:02,191 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d, entries=150, sequenceid=664, filesize=12.0 K 2024-12-02T13:18:02,192 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for f8f6542858c1817d9eb99a42a221bf04 in 1373ms, sequenceid=664, compaction requested=false 2024-12-02T13:18:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-02T13:18:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-02T13:18:02,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T13:18:02,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5270 sec 2024-12-02T13:18:02,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.5320 sec 2024-12-02T13:18:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T13:18:02,814 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:02,814 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-02T13:18:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:02,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:02,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:02,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:02,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-02T13:18:02,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:02,970 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:02,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ced0df98c6644a1cbe83b91a203a6402_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145480981/Put/seqid=0 2024-12-02T13:18:02,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742043_1219 (size=12454) 2024-12-02T13:18:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:03,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:03,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145483170, max timestamp=1733145483184Num uniques 859; 2024-12-02T13:18:03,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:03,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145483170, max timestamp=1733145483184Num uniques 859; 2024-12-02T13:18:03,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:03,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145483170, max timestamp=1733145483184Num uniques 859; 2024-12-02T13:18:03,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:03,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145543212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145543215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145543325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145543325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50506 deadline: 1733145543364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,369 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in f8f6542858c1817d9eb99a42a221bf04 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:03,389 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ced0df98c6644a1cbe83b91a203a6402_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ced0df98c6644a1cbe83b91a203a6402_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/a8fe72289fe74c7ba211130a8c48d7fb, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/a8fe72289fe74c7ba211130a8c48d7fb is 175, key is test_row_0/A:col10/1733145480981/Put/seqid=0 2024-12-02T13:18:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742044_1220 (size=31255) 2024-12-02T13:18:03,419 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=678, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/a8fe72289fe74c7ba211130a8c48d7fb 2024-12-02T13:18:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f3f588c425404992a8740a46f08a4165 is 50, key is test_row_0/B:col10/1733145480981/Put/seqid=0 2024-12-02T13:18:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742045_1221 (size=12301) 2024-12-02T13:18:03,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145543534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145543535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,851 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f3f588c425404992a8740a46f08a4165 2024-12-02T13:18:03,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145543854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145543854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:03,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2352520acfdb425e91388ff64f0e65cd is 50, key is test_row_0/C:col10/1733145480981/Put/seqid=0 2024-12-02T13:18:03,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742046_1222 (size=12301) 2024-12-02T13:18:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:04,267 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=678 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2352520acfdb425e91388ff64f0e65cd 2024-12-02T13:18:04,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/a8fe72289fe74c7ba211130a8c48d7fb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb 2024-12-02T13:18:04,277 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb, entries=150, sequenceid=678, filesize=30.5 K 2024-12-02T13:18:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/f3f588c425404992a8740a46f08a4165 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165 2024-12-02T13:18:04,282 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165, entries=150, sequenceid=678, filesize=12.0 K 2024-12-02T13:18:04,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/2352520acfdb425e91388ff64f0e65cd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd 2024-12-02T13:18:04,288 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd, entries=150, sequenceid=678, filesize=12.0 K 2024-12-02T13:18:04,289 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f8f6542858c1817d9eb99a42a221bf04 in 1319ms, sequenceid=678, compaction requested=true 2024-12-02T13:18:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-02T13:18:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-02T13:18:04,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T13:18:04,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-12-02T13:18:04,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4790 sec 2024-12-02T13:18:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:04,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:04,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120229d81cb2b9e14d4d9ab472b447c7bc24_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742047_1223 (size=14994) 2024-12-02T13:18:04,381 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145544381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145544382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120229d81cb2b9e14d4d9ab472b447c7bc24_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120229d81cb2b9e14d4d9ab472b447c7bc24_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:04,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/172df02ac8e64fbaaa5d7bfe91a73bda, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/172df02ac8e64fbaaa5d7bfe91a73bda is 175, key is test_row_0/A:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742048_1224 (size=39949) 2024-12-02T13:18:04,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=714, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/172df02ac8e64fbaaa5d7bfe91a73bda 2024-12-02T13:18:04,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eac049e669694aabbc716ae481e9248a is 50, key is test_row_0/B:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742049_1225 (size=12301) 2024-12-02T13:18:04,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eac049e669694aabbc716ae481e9248a 2024-12-02T13:18:04,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/761384a0396843dab023200bfc3221be is 50, key is test_row_0/C:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742050_1226 (size=12301) 2024-12-02T13:18:04,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/761384a0396843dab023200bfc3221be 2024-12-02T13:18:04,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/172df02ac8e64fbaaa5d7bfe91a73bda as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda 2024-12-02T13:18:04,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda, entries=200, sequenceid=714, filesize=39.0 K 2024-12-02T13:18:04,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/eac049e669694aabbc716ae481e9248a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a 2024-12-02T13:18:04,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a, entries=150, sequenceid=714, filesize=12.0 K 2024-12-02T13:18:04,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/761384a0396843dab023200bfc3221be as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be 2024-12-02T13:18:04,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be, entries=150, sequenceid=714, filesize=12.0 K 2024-12-02T13:18:04,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f8f6542858c1817d9eb99a42a221bf04 in 98ms, sequenceid=714, compaction requested=true 2024-12-02T13:18:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:04,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:04,467 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:04,468 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f6542858c1817d9eb99a42a221bf04:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:04,469 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:04,469 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/A is initiating minor compaction (all files) 2024-12-02T13:18:04,469 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/A in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:04,469 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=131.4 K 2024-12-02T13:18:04,470 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda] 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/B is initiating minor compaction (all files) 2024-12-02T13:18:04,470 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/B in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:04,470 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af6a9fbbab5b483881b1186070d176be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=48.8 K 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting af6a9fbbab5b483881b1186070d176be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479126 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting caca07f53d2f4abd9cc73cd3269c185f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479126 2024-12-02T13:18:04,470 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1889d6093a74cc1a5c9960728c88497, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733145480336 2024-12-02T13:18:04,471 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eecccb1a634b4417b4d90c5ceacfba5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733145480336 2024-12-02T13:18:04,471 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fe72289fe74c7ba211130a8c48d7fb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733145480981 2024-12-02T13:18:04,471 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f588c425404992a8740a46f08a4165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733145480981 2024-12-02T13:18:04,471 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172df02ac8e64fbaaa5d7bfe91a73bda, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733145483207 2024-12-02T13:18:04,471 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting eac049e669694aabbc716ae481e9248a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733145483213 2024-12-02T13:18:04,481 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,482 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:04,483 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/14101b5b99a343fc82535d1e2ae95683 is 50, key is test_row_0/B:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:04,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:04,503 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412023083439e30e54fa79f7342fb83d93a77_f8f6542858c1817d9eb99a42a221bf04 store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,505 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412023083439e30e54fa79f7342fb83d93a77_f8f6542858c1817d9eb99a42a221bf04, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,505 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023083439e30e54fa79f7342fb83d93a77_f8f6542858c1817d9eb99a42a221bf04 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742051_1227 (size=13255) 2024-12-02T13:18:04,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028dbde685bb68497fa092a3525d9c0c94_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145484380/Put/seqid=0 2024-12-02T13:18:04,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145484534, max timestamp=1733145484552Num uniques 859; 2024-12-02T13:18:04,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:04,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145484534, max timestamp=1733145484552Num uniques 859; 2024-12-02T13:18:04,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:04,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145484534, max timestamp=1733145484552Num uniques 859; 2024-12-02T13:18:04,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:04,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742052_1228 (size=4469) 2024-12-02T13:18:04,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742053_1229 (size=14994) 2024-12-02T13:18:04,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145544607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 360 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145544609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145544713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 361 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145544724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,921 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/14101b5b99a343fc82535d1e2ae95683 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/14101b5b99a343fc82535d1e2ae95683 2024-12-02T13:18:04,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145544925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,930 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/B of f8f6542858c1817d9eb99a42a221bf04 into 14101b5b99a343fc82535d1e2ae95683(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:04,930 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:04,930 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/B, priority=12, startTime=1733145484468; duration=0sec 2024-12-02T13:18:04,930 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:04,930 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:B 2024-12-02T13:18:04,930 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:04,931 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:04,931 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): f8f6542858c1817d9eb99a42a221bf04/C is initiating minor compaction (all files) 2024-12-02T13:18:04,932 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f6542858c1817d9eb99a42a221bf04/C in TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:04,932 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3871d0051aa74c7793a313c9119c58b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp, totalSize=48.8 K 2024-12-02T13:18:04,932 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3871d0051aa74c7793a313c9119c58b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=630, earliestPutTs=1733145479126 2024-12-02T13:18:04,933 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e412a6143e5d45c99eb4b4c26fadff1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1733145480336 2024-12-02T13:18:04,934 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2352520acfdb425e91388ff64f0e65cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=678, earliestPutTs=1733145480981 2024-12-02T13:18:04,934 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 761384a0396843dab023200bfc3221be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733145483213 2024-12-02T13:18:04,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:04,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 362 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145544934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:04,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T13:18:04,964 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:04,965 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:04,965 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:04,965 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f6542858c1817d9eb99a42a221bf04#A#compaction#184 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-02T13:18:04,966 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/a9234b7bdb0849119080b62d94c96caa is 50, key is test_row_0/C:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,966 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/4a4380c8cb6d407f9c084f7f17f2e40e is 175, key is test_row_0/A:col10/1733145483213/Put/seqid=0 2024-12-02T13:18:04,967 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:04,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:04,967 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:04,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:04,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028dbde685bb68497fa092a3525d9c0c94_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dbde685bb68497fa092a3525d9c0c94_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:04,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/cb45fa0ccaec471ba490c1701d9c4585, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:04,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/cb45fa0ccaec471ba490c1701d9c4585 is 175, key is test_row_0/A:col10/1733145484380/Put/seqid=0 2024-12-02T13:18:04,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742054_1230 (size=32209) 2024-12-02T13:18:05,013 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-12-02T13:18:05,013 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:05,013 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,014 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:05,019 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-02T13:18:05,019 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:05,019 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,019 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:05,026 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-12-02T13:18:05,026 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:05,026 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,026 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:05,026 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-02T13:18:05,026 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:05,026 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,026 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:05,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742056_1232 (size=13255) 2024-12-02T13:18:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742055_1231 (size=39949) 2024-12-02T13:18:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:05,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145545234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:05,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145545243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:05,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:05,402 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/4a4380c8cb6d407f9c084f7f17f2e40e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/4a4380c8cb6d407f9c084f7f17f2e40e 2024-12-02T13:18:05,408 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/A of f8f6542858c1817d9eb99a42a221bf04 into 4a4380c8cb6d407f9c084f7f17f2e40e(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:05,408 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:05,408 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/A, priority=12, startTime=1733145484467; duration=0sec 2024-12-02T13:18:05,409 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:05,409 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:A 2024-12-02T13:18:05,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=724, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/cb45fa0ccaec471ba490c1701d9c4585 2024-12-02T13:18:05,455 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/a9234b7bdb0849119080b62d94c96caa as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a9234b7bdb0849119080b62d94c96caa 2024-12-02T13:18:05,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c515e389a0d040689bd3c59d63896ef0 is 50, key is test_row_0/B:col10/1733145484380/Put/seqid=0 2024-12-02T13:18:05,461 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f8f6542858c1817d9eb99a42a221bf04/C of f8f6542858c1817d9eb99a42a221bf04 into a9234b7bdb0849119080b62d94c96caa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:05,461 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:05,461 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., storeName=f8f6542858c1817d9eb99a42a221bf04/C, priority=12, startTime=1733145484468; duration=0sec 2024-12-02T13:18:05,461 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:05,461 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f6542858c1817d9eb99a42a221bf04:C 2024-12-02T13:18:05,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742057_1233 (size=12301) 2024-12-02T13:18:05,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:05,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50548 deadline: 1733145545744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:05,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50532 deadline: 1733145545754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:05,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c515e389a0d040689bd3c59d63896ef0 2024-12-02T13:18:05,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ac6a045f83242638cc079873f8e4fbf is 50, key is test_row_0/C:col10/1733145484380/Put/seqid=0 2024-12-02T13:18:05,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742058_1234 (size=12301) 2024-12-02T13:18:05,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:05,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:05,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:05,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:06,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:06,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:06,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:06,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:06,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:06,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ac6a045f83242638cc079873f8e4fbf 2024-12-02T13:18:06,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/cb45fa0ccaec471ba490c1701d9c4585 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/cb45fa0ccaec471ba490c1701d9c4585 2024-12-02T13:18:06,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/cb45fa0ccaec471ba490c1701d9c4585, entries=200, sequenceid=724, filesize=39.0 K 2024-12-02T13:18:06,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/c515e389a0d040689bd3c59d63896ef0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c515e389a0d040689bd3c59d63896ef0 2024-12-02T13:18:06,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c515e389a0d040689bd3c59d63896ef0, entries=150, sequenceid=724, filesize=12.0 K 2024-12-02T13:18:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/0ac6a045f83242638cc079873f8e4fbf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ac6a045f83242638cc079873f8e4fbf 2024-12-02T13:18:06,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ac6a045f83242638cc079873f8e4fbf, entries=150, sequenceid=724, filesize=12.0 K 2024-12-02T13:18:06,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f8f6542858c1817d9eb99a42a221bf04 in 1803ms, sequenceid=724, compaction requested=false 2024-12-02T13:18:06,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:06,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-02T13:18:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:06,346 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:06,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f9237ef4a0548e7b8adae7251f71fc3_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145484607/Put/seqid=0 2024-12-02T13:18:06,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742059_1235 (size=12454) 2024-12-02T13:18:06,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:06,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. as already flushing 2024-12-02T13:18:06,755 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-12-02T13:18:06,755 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:06,755 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:06,755 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:06,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:06,758 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022f9237ef4a0548e7b8adae7251f71fc3_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9237ef4a0548e7b8adae7251f71fc3_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:06,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/7e327c389cb04317bb4e499269d933b9, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:06,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/7e327c389cb04317bb4e499269d933b9 is 175, key is test_row_0/A:col10/1733145484607/Put/seqid=0 2024-12-02T13:18:06,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742060_1236 (size=31255) 2024-12-02T13:18:06,765 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-12-02T13:18:06,765 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:06,765 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:06,765 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:06,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:06,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:18:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:07,162 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=763, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/7e327c389cb04317bb4e499269d933b9 2024-12-02T13:18:07,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a550d533f3de4d11ba411c6bb14a9f6e is 50, key is test_row_0/B:col10/1733145484607/Put/seqid=0 2024-12-02T13:18:07,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742061_1237 (size=12301) 2024-12-02T13:18:07,179 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a550d533f3de4d11ba411c6bb14a9f6e 2024-12-02T13:18:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/10cfd4dc46ff4892a063200e8cd51e26 is 50, key is test_row_0/C:col10/1733145484607/Put/seqid=0 2024-12-02T13:18:07,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742062_1238 (size=12301) 2024-12-02T13:18:07,445 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-12-02T13:18:07,445 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:07,445 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:07,445 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:07,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:07,525 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-12-02T13:18:07,525 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:07,525 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:07,525 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:07,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:07,592 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/10cfd4dc46ff4892a063200e8cd51e26 2024-12-02T13:18:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/7e327c389cb04317bb4e499269d933b9 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/7e327c389cb04317bb4e499269d933b9 2024-12-02T13:18:07,600 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/7e327c389cb04317bb4e499269d933b9, entries=150, sequenceid=763, filesize=30.5 K 2024-12-02T13:18:07,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/a550d533f3de4d11ba411c6bb14a9f6e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a550d533f3de4d11ba411c6bb14a9f6e 2024-12-02T13:18:07,604 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a550d533f3de4d11ba411c6bb14a9f6e, entries=150, sequenceid=763, filesize=12.0 K 2024-12-02T13:18:07,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/10cfd4dc46ff4892a063200e8cd51e26 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/10cfd4dc46ff4892a063200e8cd51e26 2024-12-02T13:18:07,609 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/10cfd4dc46ff4892a063200e8cd51e26, entries=150, sequenceid=763, filesize=12.0 K 2024-12-02T13:18:07,610 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=26.84 KB/27480 for f8f6542858c1817d9eb99a42a221bf04 in 1265ms, sequenceid=763, compaction requested=true 2024-12-02T13:18:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f8f6542858c1817d9eb99a42a221bf04: 2024-12-02T13:18:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-02T13:18:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-02T13:18:07,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T13:18:07,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-12-02T13:18:07,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.6480 sec 2024-12-02T13:18:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-02T13:18:09,124 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:13,455 INFO [Thread-595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-595. 2024-12-02T13:18:13,456 DEBUG [Thread-595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:13,456 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:13,456 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:13,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 299 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 258 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3141 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3037 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1366 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4098 rows 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1369 2024-12-02T13:18:13,456 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4102 rows 2024-12-02T13:18:13,456 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:18:13,456 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:18:13,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:13,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:13,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:13,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:18:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:13,459 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145493459"}]},"ts":"1733145493459"} 2024-12-02T13:18:13,461 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:18:13,461 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:18:13,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:18:13,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, UNASSIGN}] 2024-12-02T13:18:13,463 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, UNASSIGN 2024-12-02T13:18:13,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:13,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, UNASSIGN because future has completed 2024-12-02T13:18:13,465 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:18:13,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:13,617 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing f8f6542858c1817d9eb99a42a221bf04, disabling compactions & flushes 2024-12-02T13:18:13,617 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. after waiting 0 ms 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:13,617 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing f8f6542858c1817d9eb99a42a221bf04 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=A 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=B 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f6542858c1817d9eb99a42a221bf04, store=C 2024-12-02T13:18:13,617 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:13,622 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d192eb703acf4268a9fc63e14c52f071_f8f6542858c1817d9eb99a42a221bf04 is 50, key is test_row_0/A:col10/1733145493454/Put/seqid=0 2024-12-02T13:18:13,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742063_1239 (size=9914) 2024-12-02T13:18:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:14,026 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:14,029 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d192eb703acf4268a9fc63e14c52f071_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d192eb703acf4268a9fc63e14c52f071_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:14,030 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d58e27b7d1c74a62bf62847c729b2efb, store: [table=TestAcidGuarantees family=A region=f8f6542858c1817d9eb99a42a221bf04] 2024-12-02T13:18:14,031 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d58e27b7d1c74a62bf62847c729b2efb is 175, key is test_row_0/A:col10/1733145493454/Put/seqid=0 2024-12-02T13:18:14,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742064_1240 (size=22561) 2024-12-02T13:18:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:14,439 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=771, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d58e27b7d1c74a62bf62847c729b2efb 2024-12-02T13:18:14,446 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/af8554d32d8a42f6822dbaafe5b46095 is 50, key is test_row_0/B:col10/1733145493454/Put/seqid=0 2024-12-02T13:18:14,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742065_1241 (size=9857) 2024-12-02T13:18:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:14,850 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/af8554d32d8a42f6822dbaafe5b46095 2024-12-02T13:18:14,856 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/c3a3e588aa944cb4bf17761d63029180 is 50, key is test_row_0/C:col10/1733145493454/Put/seqid=0 2024-12-02T13:18:14,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742066_1242 (size=9857) 2024-12-02T13:18:15,261 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=771 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/c3a3e588aa944cb4bf17761d63029180 2024-12-02T13:18:15,265 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/A/d58e27b7d1c74a62bf62847c729b2efb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d58e27b7d1c74a62bf62847c729b2efb 2024-12-02T13:18:15,268 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d58e27b7d1c74a62bf62847c729b2efb, entries=100, sequenceid=771, filesize=22.0 K 2024-12-02T13:18:15,269 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/B/af8554d32d8a42f6822dbaafe5b46095 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af8554d32d8a42f6822dbaafe5b46095 2024-12-02T13:18:15,273 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af8554d32d8a42f6822dbaafe5b46095, entries=100, sequenceid=771, filesize=9.6 K 2024-12-02T13:18:15,273 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/.tmp/C/c3a3e588aa944cb4bf17761d63029180 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/c3a3e588aa944cb4bf17761d63029180 2024-12-02T13:18:15,277 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/c3a3e588aa944cb4bf17761d63029180, entries=100, sequenceid=771, filesize=9.6 K 2024-12-02T13:18:15,277 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8f6542858c1817d9eb99a42a221bf04 in 1660ms, sequenceid=771, compaction requested=true 2024-12-02T13:18:15,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda] to archive 2024-12-02T13:18:15,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:15,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1d7278ce799d4bcca3f286e0dffb2052 2024-12-02T13:18:15,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3fc2e2de07a74ca3a663f26442fd7ca2 2024-12-02T13:18:15,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c4b07b3cb40049fc8333ec60eae614be 2024-12-02T13:18:15,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/28ed30c4bb0546708ed23266bd5617de 2024-12-02T13:18:15,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/9064544987734d6988c2dd7ab3eb8785 2024-12-02T13:18:15,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d0e65e6992684b9ca15e4c348912c894 2024-12-02T13:18:15,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/37fbfa9ed5a4490590fe9554e8cb1cc5 2024-12-02T13:18:15,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/0fee35964d504510987a5cd95442d6bb 2024-12-02T13:18:15,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/59e70df6727e4d858df9a9ce61b75f90 2024-12-02T13:18:15,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66eda8fe1e824e70949800f1b06cf3f7 2024-12-02T13:18:15,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/b3b7043d4b684ad1bb8933991494fd23 2024-12-02T13:18:15,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ce5a4ef226be44a3ab10fca8b7b5b3cc 2024-12-02T13:18:15,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/3b9e93b0dc96497cbc64e39e8485392f 2024-12-02T13:18:15,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c81606aed8bd48078a99fe755421d8a8 2024-12-02T13:18:15,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/c005444843724c418592e18e33fd1097 2024-12-02T13:18:15,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/66038be181294f99b4e9969d41b381db 2024-12-02T13:18:15,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/96d38517f4f542aa8c59fb5dc0e7a314 2024-12-02T13:18:15,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/591b4f92bab346fea7394d4ee230f844 2024-12-02T13:18:15,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c2b28a7924d4e65afe7946129bbb26c 2024-12-02T13:18:15,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/ea90ef56638847f2a5a70ccb8818b17b 2024-12-02T13:18:15,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/88ece19fc8cf46dcb180a1bfb19fc1d6 2024-12-02T13:18:15,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/af3d5696be3c4bca979a73108e6030fd 2024-12-02T13:18:15,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1c825dc7587141c49bce9882af0e3861 2024-12-02T13:18:15,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/1aa8646b18e44762b6e698fcfe7a538d 2024-12-02T13:18:15,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/caca07f53d2f4abd9cc73cd3269c185f 2024-12-02T13:18:15,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/f1889d6093a74cc1a5c9960728c88497 2024-12-02T13:18:15,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/a8fe72289fe74c7ba211130a8c48d7fb 2024-12-02T13:18:15,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/172df02ac8e64fbaaa5d7bfe91a73bda 2024-12-02T13:18:15,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/2c4ffdee6d7f4f2d9972f2f277296fdc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/233b8195ebd346f2af0c0da4601ebf66, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e50cbb77a8e74c9990293ed1153dec4d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a44c5fbee47e484bb664d0012a48905d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a0f4f7c679164e62b1f56220761d45ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/694e03ce75dd4d82acdf6519ecbada3a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af6a9fbbab5b483881b1186070d176be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a] to archive 2024-12-02T13:18:15,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:15,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/557d2eaf16214fcca09d5e7fcaf2a313 2024-12-02T13:18:15,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a3a28eb2753d49e89bec26f8cbeb9bcc 2024-12-02T13:18:15,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/2c4ffdee6d7f4f2d9972f2f277296fdc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/2c4ffdee6d7f4f2d9972f2f277296fdc 2024-12-02T13:18:15,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/8655b8483bbb42b9ae6c3f4fdd7c65b2 2024-12-02T13:18:15,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/570ea49f38de4e318690b156595c20dd 2024-12-02T13:18:15,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/233b8195ebd346f2af0c0da4601ebf66 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/233b8195ebd346f2af0c0da4601ebf66 2024-12-02T13:18:15,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/44ae809c4d92402f8926d0310671aa4b 2024-12-02T13:18:15,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a10aebe237ab441cbc534d39a19b3a5d 2024-12-02T13:18:15,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e50cbb77a8e74c9990293ed1153dec4d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e50cbb77a8e74c9990293ed1153dec4d 2024-12-02T13:18:15,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e395714578cd4be28d429d91bb89ce8f 2024-12-02T13:18:15,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/54b6c141038d45b787b0cc144d0b0497 2024-12-02T13:18:15,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a44c5fbee47e484bb664d0012a48905d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a44c5fbee47e484bb664d0012a48905d 2024-12-02T13:18:15,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/30adc20a10f04dfc97e22c165ff1ad12 2024-12-02T13:18:15,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/05b389d916cd4643982673a436e3b063 2024-12-02T13:18:15,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a0f4f7c679164e62b1f56220761d45ec to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a0f4f7c679164e62b1f56220761d45ec 2024-12-02T13:18:15,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/b30476e01da94e24b354ef49fd6f820c 2024-12-02T13:18:15,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/899e97234e674c8e898c2876cfae217e 2024-12-02T13:18:15,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c7e8fe478aff42ed874dd943631024a9 2024-12-02T13:18:15,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/e8d062bb4d6544d48542f6162fc7a6bc 2024-12-02T13:18:15,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/694e03ce75dd4d82acdf6519ecbada3a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/694e03ce75dd4d82acdf6519ecbada3a 2024-12-02T13:18:15,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a881fdfe305b4b0c8a5863cf32b11e9d 2024-12-02T13:18:15,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f4758ae1eb814bba9aabda5584b9b0ac 2024-12-02T13:18:15,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/335623f41965410396db34997c6625ad 2024-12-02T13:18:15,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af6a9fbbab5b483881b1186070d176be to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af6a9fbbab5b483881b1186070d176be 2024-12-02T13:18:15,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f38c361affdd45bab005ef41e29aa563 2024-12-02T13:18:15,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eecccb1a634b4417b4d90c5ceacfba5f 2024-12-02T13:18:15,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/f3f588c425404992a8740a46f08a4165 2024-12-02T13:18:15,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/eac049e669694aabbc716ae481e9248a 2024-12-02T13:18:15,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ea57b015324946adbf6f4dc88c1c237b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3cb8507e32cf43039d2622f3a0eb5284, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/bb41c0d8578f434a8fc130146704f906, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/230cd8af0cd741f683647b61357b38be, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2fe2061028b7440e90d956fc9c5a8ff4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/43f5981f28d04183be1a636e04073e26, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3871d0051aa74c7793a313c9119c58b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be] to archive 2024-12-02T13:18:15,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:15,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1c0e77fd269f4db68094ae9e14b13a18 2024-12-02T13:18:15,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/5e8cd205dd144ea8ba7ab72c73d104ae 2024-12-02T13:18:15,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ea57b015324946adbf6f4dc88c1c237b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ea57b015324946adbf6f4dc88c1c237b 2024-12-02T13:18:15,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f2d7cd3432c14dcea400ad944e6a4acf 2024-12-02T13:18:15,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/608cafcdb68c4ef1813bdf163ba09ed1 2024-12-02T13:18:15,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3cb8507e32cf43039d2622f3a0eb5284 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3cb8507e32cf43039d2622f3a0eb5284 2024-12-02T13:18:15,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1a6adb332be44aecbe811ed7344d38dd 2024-12-02T13:18:15,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/1e762e15bb7346dea17cb56284e26342 2024-12-02T13:18:15,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/bb41c0d8578f434a8fc130146704f906 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/bb41c0d8578f434a8fc130146704f906 2024-12-02T13:18:15,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f8dc165b129b47ec8d7995678e63dff6 2024-12-02T13:18:15,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/cc8c4d26914d499baa43f8eb44ab2d25 2024-12-02T13:18:15,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/230cd8af0cd741f683647b61357b38be to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/230cd8af0cd741f683647b61357b38be 2024-12-02T13:18:15,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/25dbece239f848388295c83d3b096be6 2024-12-02T13:18:15,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ddacce6a69f4f3a92a5d8e6c8bbe57c 2024-12-02T13:18:15,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2fe2061028b7440e90d956fc9c5a8ff4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2fe2061028b7440e90d956fc9c5a8ff4 2024-12-02T13:18:15,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/ee24397bdb514ce384793a47ab53b543 2024-12-02T13:18:15,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2733e54761b54b7593387a2d55d75fe3 2024-12-02T13:18:15,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b6864d3da6d740c799cd6d4473d4f2f5 2024-12-02T13:18:15,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a3d188b401504f5d9ff32ccf6d52e600 2024-12-02T13:18:15,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/43f5981f28d04183be1a636e04073e26 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/43f5981f28d04183be1a636e04073e26 2024-12-02T13:18:15,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/b8b73be2b9b94fe9b8b1ba316e5fa170 2024-12-02T13:18:15,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/f30fed3e439c4b33b67b5b638eac4a6d 2024-12-02T13:18:15,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/daeed142fdb64776a383c282db436d6f 2024-12-02T13:18:15,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3871d0051aa74c7793a313c9119c58b6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/3871d0051aa74c7793a313c9119c58b6 2024-12-02T13:18:15,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2e8e8a151c8a4a2892b352a3e7fdb575 2024-12-02T13:18:15,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/e412a6143e5d45c99eb4b4c26fadff1d 2024-12-02T13:18:15,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/2352520acfdb425e91388ff64f0e65cd 2024-12-02T13:18:15,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/761384a0396843dab023200bfc3221be 2024-12-02T13:18:15,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits/774.seqid, newMaxSeqId=774, maxSeqId=4 2024-12-02T13:18:15,366 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. 2024-12-02T13:18:15,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for f8f6542858c1817d9eb99a42a221bf04: Waiting for close lock at 1733145493617Running coprocessor pre-close hooks at 1733145493617Disabling compacts and flushes for region at 1733145493617Disabling writes for close at 1733145493617Obtaining lock to block concurrent updates at 1733145493617Preparing flush snapshotting stores in f8f6542858c1817d9eb99a42a221bf04 at 1733145493617Finished memstore snapshotting TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733145493618 (+1 ms)Flushing stores of TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04. at 1733145493618Flushing f8f6542858c1817d9eb99a42a221bf04/A: creating writer at 1733145493618Flushing mob file f8f6542858c1817d9eb99a42a221bf04/A: appending metadata at 1733145493622 (+4 ms)Flushing mob file f8f6542858c1817d9eb99a42a221bf04/A: closing flushed file at 1733145493622Flushing f8f6542858c1817d9eb99a42a221bf04/A: appending metadata at 1733145494030 (+408 ms)Flushing f8f6542858c1817d9eb99a42a221bf04/A: closing flushed file at 1733145494030Flushing f8f6542858c1817d9eb99a42a221bf04/B: creating writer at 1733145494443 (+413 ms)Flushing f8f6542858c1817d9eb99a42a221bf04/B: appending metadata at 1733145494445 (+2 ms)Flushing f8f6542858c1817d9eb99a42a221bf04/B: closing flushed file at 1733145494445Flushing f8f6542858c1817d9eb99a42a221bf04/C: creating writer at 1733145494854 (+409 ms)Flushing f8f6542858c1817d9eb99a42a221bf04/C: appending metadata at 1733145494856 (+2 ms)Flushing f8f6542858c1817d9eb99a42a221bf04/C: closing flushed file at 1733145494856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d495221: reopening flushed file at 1733145495264 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@551eca1e: reopening flushed file at 1733145495268 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c9c0d98: reopening flushed file at 1733145495273 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f8f6542858c1817d9eb99a42a221bf04 in 1660ms, sequenceid=771, compaction requested=true at 1733145495277 (+4 ms)Writing region close event to WAL at 1733145495362 (+85 ms)Running coprocessor post-close hooks at 1733145495366 (+4 ms)Closed at 1733145495366 2024-12-02T13:18:15,368 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,369 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=f8f6542858c1817d9eb99a42a221bf04, regionState=CLOSED 2024-12-02T13:18:15,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:15,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T13:18:15,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure f8f6542858c1817d9eb99a42a221bf04, server=06e3e18f2515,45693,1733145428280 in 1.9060 sec 2024-12-02T13:18:15,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T13:18:15,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f6542858c1817d9eb99a42a221bf04, UNASSIGN in 1.9100 sec 2024-12-02T13:18:15,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-02T13:18:15,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9130 sec 2024-12-02T13:18:15,376 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145495376"}]},"ts":"1733145495376"} 2024-12-02T13:18:15,377 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:18:15,378 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:18:15,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-02T13:18:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T13:18:15,604 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:15,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:18:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,606 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T13:18:15,606 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,608 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits] 2024-12-02T13:18:15,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/4a4380c8cb6d407f9c084f7f17f2e40e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/4a4380c8cb6d407f9c084f7f17f2e40e 2024-12-02T13:18:15,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/7e327c389cb04317bb4e499269d933b9 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/7e327c389cb04317bb4e499269d933b9 2024-12-02T13:18:15,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/cb45fa0ccaec471ba490c1701d9c4585 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/cb45fa0ccaec471ba490c1701d9c4585 2024-12-02T13:18:15,616 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d58e27b7d1c74a62bf62847c729b2efb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/A/d58e27b7d1c74a62bf62847c729b2efb 2024-12-02T13:18:15,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/14101b5b99a343fc82535d1e2ae95683 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/14101b5b99a343fc82535d1e2ae95683 2024-12-02T13:18:15,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a550d533f3de4d11ba411c6bb14a9f6e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/a550d533f3de4d11ba411c6bb14a9f6e 2024-12-02T13:18:15,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af8554d32d8a42f6822dbaafe5b46095 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/af8554d32d8a42f6822dbaafe5b46095 2024-12-02T13:18:15,621 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c515e389a0d040689bd3c59d63896ef0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/B/c515e389a0d040689bd3c59d63896ef0 2024-12-02T13:18:15,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ac6a045f83242638cc079873f8e4fbf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/0ac6a045f83242638cc079873f8e4fbf 2024-12-02T13:18:15,624 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/10cfd4dc46ff4892a063200e8cd51e26 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/10cfd4dc46ff4892a063200e8cd51e26 2024-12-02T13:18:15,625 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a9234b7bdb0849119080b62d94c96caa to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/a9234b7bdb0849119080b62d94c96caa 2024-12-02T13:18:15,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/c3a3e588aa944cb4bf17761d63029180 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/C/c3a3e588aa944cb4bf17761d63029180 2024-12-02T13:18:15,628 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits/774.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04/recovered.edits/774.seqid 2024-12-02T13:18:15,629 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,629 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:18:15,629 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:18:15,630 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T13:18:15,633 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020314e060208d4fa8acc25b31c46fb1c6_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020314e060208d4fa8acc25b31c46fb1c6_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022320e59d7b1f4c30af06a38e0eb8f645_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022320e59d7b1f4c30af06a38e0eb8f645_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120229d81cb2b9e14d4d9ab472b447c7bc24_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120229d81cb2b9e14d4d9ab472b447c7bc24_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f5cb292a1d64ae8b4283c39b667e58b_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f5cb292a1d64ae8b4283c39b667e58b_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9237ef4a0548e7b8adae7251f71fc3_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9237ef4a0548e7b8adae7251f71fc3_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9f1861ec2e4f2484dc437c659bddbd_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022f9f1861ec2e4f2484dc437c659bddbd_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240b326830840433c8f4cad349a13bccb_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120240b326830840433c8f4cad349a13bccb_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024363a2bec5e648b6ab51a0287f8826d0_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024363a2bec5e648b6ab51a0287f8826d0_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024b6af0340b10465781f862904e2cead6_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024b6af0340b10465781f862904e2cead6_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025556968d82104987b1090bc4ace3e73c_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025556968d82104987b1090bc4ace3e73c_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f952a4b63bf4056af286912e6e44387_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f952a4b63bf4056af286912e6e44387_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120285282118ce5f4ecd8f183340a69b6f56_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120285282118ce5f4ecd8f183340a69b6f56_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dbde685bb68497fa092a3525d9c0c94_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028dbde685bb68497fa092a3525d9c0c94_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120291b5025f68c64ec1b93d02271c5006d9_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120291b5025f68c64ec1b93d02271c5006d9_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b7c2e4d838374389ad82386e808f1e67_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b7c2e4d838374389ad82386e808f1e67_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,648 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba2c11c10c454102be5b2b9e23498148_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba2c11c10c454102be5b2b9e23498148_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bc44a277a23f4e82b9cbe5b4a4722e46_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bc44a277a23f4e82b9cbe5b4a4722e46_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca004cf8c7364a788bd6011e1d7cacf3_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ca004cf8c7364a788bd6011e1d7cacf3_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ced0df98c6644a1cbe83b91a203a6402_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ced0df98c6644a1cbe83b91a203a6402_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d192eb703acf4268a9fc63e14c52f071_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d192eb703acf4268a9fc63e14c52f071_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d2646b54a4254ead836633cad49c91ce_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d2646b54a4254ead836633cad49c91ce_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d5a52ae988184dceab9050cd9e1f47da_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d5a52ae988184dceab9050cd9e1f47da_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dcf07d0bad6040aa832dd01bfc553d16_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dcf07d0bad6040aa832dd01bfc553d16_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,655 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f3eeb48b4d54433591741e3bc41fc683_f8f6542858c1817d9eb99a42a221bf04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f3eeb48b4d54433591741e3bc41fc683_f8f6542858c1817d9eb99a42a221bf04 2024-12-02T13:18:15,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:18:15,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,660 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:18:15,662 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:18:15,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:18:15,663 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145495663"}]},"ts":"9223372036854775807"} 2024-12-02T13:18:15,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:18:15,665 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f8f6542858c1817d9eb99a42a221bf04, NAME => 'TestAcidGuarantees,,1733145461813.f8f6542858c1817d9eb99a42a221bf04.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:18:15,665 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:18:15,665 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145495665"}]},"ts":"9223372036854775807"} 2024-12-02T13:18:15,667 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:18:15,667 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-12-02T13:18:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T13:18:15,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:18:15,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:15,722 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_740994727_22 at /127.0.0.1:56318 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=512 (was 461) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1490 (was 1806) 2024-12-02T13:18:15,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=512, ProcessCount=11, AvailableMemoryMB=1490 2024-12-02T13:18:15,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:18:15,731 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:18:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:15,732 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:18:15,733 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:15,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-12-02T13:18:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:15,733 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:18:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742067_1243 (size=1061) 2024-12-02T13:18:15,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:16,139 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:18:16,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742068_1244 (size=53) 2024-12-02T13:18:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b75af96915a684ad1ebeb7dc1c1ba4d3, disabling compactions & flushes 2024-12-02T13:18:16,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. after waiting 0 ms 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,545 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b75af96915a684ad1ebeb7dc1c1ba4d3: Waiting for close lock at 1733145496545Disabling compacts and flushes for region at 1733145496545Disabling writes for close at 1733145496545Writing region close event to WAL at 1733145496545Closed at 1733145496545 2024-12-02T13:18:16,546 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:18:16,546 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145496546"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145496546"}]},"ts":"1733145496546"} 2024-12-02T13:18:16,547 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:18:16,548 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:18:16,548 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145496548"}]},"ts":"1733145496548"} 2024-12-02T13:18:16,549 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:18:16,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, ASSIGN}] 2024-12-02T13:18:16,550 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, ASSIGN 2024-12-02T13:18:16,551 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:18:16,701 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=b75af96915a684ad1ebeb7dc1c1ba4d3, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:16,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, ASSIGN because future has completed 2024-12-02T13:18:16,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:16,857 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,857 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:18:16,857 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,857 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:18:16,857 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,857 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,859 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,860 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:16,860 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b75af96915a684ad1ebeb7dc1c1ba4d3 columnFamilyName A 2024-12-02T13:18:16,860 DEBUG [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:16,861 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(327): Store=b75af96915a684ad1ebeb7dc1c1ba4d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:16,861 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,862 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:16,862 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b75af96915a684ad1ebeb7dc1c1ba4d3 columnFamilyName B 2024-12-02T13:18:16,862 DEBUG [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:16,863 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(327): Store=b75af96915a684ad1ebeb7dc1c1ba4d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:16,863 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,864 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:16,864 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b75af96915a684ad1ebeb7dc1c1ba4d3 columnFamilyName C 2024-12-02T13:18:16,864 DEBUG [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:16,865 INFO [StoreOpener-b75af96915a684ad1ebeb7dc1c1ba4d3-1 {}] regionserver.HStore(327): Store=b75af96915a684ad1ebeb7dc1c1ba4d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:16,865 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,866 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,866 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,867 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,867 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,867 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,868 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:18:16,869 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:16,875 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:18:16,875 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened b75af96915a684ad1ebeb7dc1c1ba4d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60842166, jitterRate=-0.09338107705116272}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:18:16,875 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:16,876 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for b75af96915a684ad1ebeb7dc1c1ba4d3: Running coprocessor pre-open hook at 1733145496858Writing region info on filesystem at 1733145496858Initializing all the Stores at 1733145496859 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145496859Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145496859Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145496859Cleaning up temporary data from old regions at 1733145496867 (+8 ms)Running coprocessor post-open hooks at 1733145496875 (+8 ms)Region opened successfully at 1733145496876 (+1 ms) 2024-12-02T13:18:16,877 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., pid=67, masterSystemTime=1733145496854 2024-12-02T13:18:16,878 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,878 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:16,879 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=b75af96915a684ad1ebeb7dc1c1ba4d3, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:16,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:16,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T13:18:16,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 in 177 msec 2024-12-02T13:18:16,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-02T13:18:16,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, ASSIGN in 333 msec 2024-12-02T13:18:16,885 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:18:16,885 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145496885"}]},"ts":"1733145496885"} 2024-12-02T13:18:16,887 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:18:16,888 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:18:16,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-12-02T13:18:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T13:18:17,884 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:17,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28720ecc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d20c4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:18:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:18:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e67ae44 2024-12-02T13:18:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:18:17,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:18:17,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559a3c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,895 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117e62c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685c3cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229e0e67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fee3b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733f95a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6e39b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fea051f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c63a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d37ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563276d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b486a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3fe818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78455341, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,921 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c343e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,924 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8ce0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1148c75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,926 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:17,926 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:17,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:17,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b98a4f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:17,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:17,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf08674, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:17,929 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,930 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,931 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T13:18:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:17,932 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,933 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,934 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:17,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,936 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,936 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,940 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,947 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,948 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,948 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:17,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,950 DEBUG [Thread-1150 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,950 DEBUG [Thread-1150 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:17,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:17,955 DEBUG [Thread-1141 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:17,955 DEBUG [Thread-1141 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:17,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:17,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:17,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:17,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145557987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:17,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/4492c53b7aac43c098390bebe7752d18 is 50, key is test_row_0/A:col10/1733145497946/Put/seqid=0 2024-12-02T13:18:17,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145557988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:17,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145497985, max timestamp=1733145497995Num uniques 859; 2024-12-02T13:18:17,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145557990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145497985, max timestamp=1733145497995Num uniques 859; 2024-12-02T13:18:18,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:18,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145497985, max timestamp=1733145497995Num uniques 859; 2024-12-02T13:18:18,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:18,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742069_1245 (size=12001) 2024-12-02T13:18:18,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145558053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145558056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145558104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145558104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145558104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145558164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:18,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145558315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145558315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145558315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145558374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145558374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/4492c53b7aac43c098390bebe7752d18 2024-12-02T13:18:18,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/78fbb9b1519a468f9ba9188c08b7c34e is 50, key is test_row_0/B:col10/1733145497946/Put/seqid=0 2024-12-02T13:18:18,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742070_1246 (size=12001) 2024-12-02T13:18:18,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:18,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145558623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145558624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145558624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145558684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145558684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:18,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:18,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:18,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/78fbb9b1519a468f9ba9188c08b7c34e 2024-12-02T13:18:18,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/4d1e2708b4bc49f3af6951f18710e833 is 50, key is test_row_0/C:col10/1733145497946/Put/seqid=0 2024-12-02T13:18:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742071_1247 (size=12001) 2024-12-02T13:18:18,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/4d1e2708b4bc49f3af6951f18710e833 2024-12-02T13:18:18,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/4492c53b7aac43c098390bebe7752d18 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18 2024-12-02T13:18:18,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:18:18,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/78fbb9b1519a468f9ba9188c08b7c34e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e 2024-12-02T13:18:18,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:18:18,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/4d1e2708b4bc49f3af6951f18710e833 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833 2024-12-02T13:18:18,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T13:18:18,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1017ms, sequenceid=13, compaction requested=false 2024-12-02T13:18:18,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:19,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T13:18:19,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:19,011 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:19,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a66f15b3f5c0488880aadcad35af349e is 50, key is test_row_0/A:col10/1733145498053/Put/seqid=0 2024-12-02T13:18:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:19,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742072_1248 (size=12001) 2024-12-02T13:18:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:19,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145559150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145559150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145559151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145559194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145559195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145559263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145559263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145559263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145559474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145559474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145559474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,491 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a66f15b3f5c0488880aadcad35af349e 2024-12-02T13:18:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/3afadfc8ac1e43f79438357cdf0750c3 is 50, key is test_row_0/B:col10/1733145498053/Put/seqid=0 2024-12-02T13:18:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742073_1249 (size=12001) 2024-12-02T13:18:19,511 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/3afadfc8ac1e43f79438357cdf0750c3 2024-12-02T13:18:19,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c0762efa64fc4549aeb643d2c479ceeb is 50, key is test_row_0/C:col10/1733145498053/Put/seqid=0 2024-12-02T13:18:19,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742074_1250 (size=12001) 2024-12-02T13:18:19,527 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c0762efa64fc4549aeb643d2c479ceeb 2024-12-02T13:18:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a66f15b3f5c0488880aadcad35af349e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e 2024-12-02T13:18:19,536 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T13:18:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/3afadfc8ac1e43f79438357cdf0750c3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3 2024-12-02T13:18:19,540 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T13:18:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c0762efa64fc4549aeb643d2c479ceeb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb 2024-12-02T13:18:19,554 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T13:18:19,555 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 545ms, sequenceid=48, compaction requested=false 2024-12-02T13:18:19,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:19,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:19,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T13:18:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T13:18:19,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T13:18:19,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-12-02T13:18:19,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6300 sec 2024-12-02T13:18:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:19,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:19,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2ed272c9a14c492589f8569b06095f3a is 50, key is test_row_0/A:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:19,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742075_1251 (size=12001) 2024-12-02T13:18:19,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2ed272c9a14c492589f8569b06095f3a 2024-12-02T13:18:19,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b4007f06e7c54136b2dce4900c9d7e0f is 50, key is test_row_0/B:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:19,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742076_1252 (size=12001) 2024-12-02T13:18:19,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b4007f06e7c54136b2dce4900c9d7e0f 2024-12-02T13:18:19,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/90b174ecdd564c33a63af75f451b4017 is 50, key is test_row_0/C:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742077_1253 (size=12001) 2024-12-02T13:18:19,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733145499806, max timestamp=1733145499845Num uniques 859; 2024-12-02T13:18:19,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:18:19,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733145499823, max timestamp=1733145499845Num uniques 859; 2024-12-02T13:18:19,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T13:18:19,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733145499823, max timestamp=1733145499845Num uniques 859; 2024-12-02T13:18:19,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T13:18:19,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145559897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145559898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:19,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145560014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145560014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145560014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T13:18:20,084 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:20,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T13:18:20,086 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:20,087 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:20,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T13:18:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T13:18:20,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145560214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145560214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,217 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,217 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145560223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145560223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145560223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T13:18:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/90b174ecdd564c33a63af75f451b4017 2024-12-02T13:18:20,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2ed272c9a14c492589f8569b06095f3a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a 2024-12-02T13:18:20,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T13:18:20,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b4007f06e7c54136b2dce4900c9d7e0f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f 2024-12-02T13:18:20,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T13:18:20,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/90b174ecdd564c33a63af75f451b4017 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017 2024-12-02T13:18:20,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T13:18:20,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=117.74 KB/120562 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 489ms, sequenceid=61, compaction requested=true 2024-12-02T13:18:20,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:20,277 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:20,277 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:20,278 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:20,278 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:20,278 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:20,278 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:20,278 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,278 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,278 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.2 K 2024-12-02T13:18:20,278 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.2 K 2024-12-02T13:18:20,278 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 78fbb9b1519a468f9ba9188c08b7c34e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145497945 2024-12-02T13:18:20,279 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4492c53b7aac43c098390bebe7752d18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145497945 2024-12-02T13:18:20,279 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afadfc8ac1e43f79438357cdf0750c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733145498044 2024-12-02T13:18:20,279 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66f15b3f5c0488880aadcad35af349e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733145498044 2024-12-02T13:18:20,279 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b4007f06e7c54136b2dce4900c9d7e0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:20,279 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed272c9a14c492589f8569b06095f3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:20,288 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:20,289 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f is 50, key is test_row_0/B:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:20,295 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:20,295 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f37e10de16bf49b99a4d519b09959e0f is 50, key is test_row_0/A:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:20,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742078_1254 (size=12104) 2024-12-02T13:18:20,316 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f 2024-12-02T13:18:20,323 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into c49f0a6c79ac4c8485b2c3c3fe4d4a2f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:20,323 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:20,323 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145500277; duration=0sec 2024-12-02T13:18:20,323 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:20,323 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:20,323 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:20,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742079_1255 (size=12104) 2024-12-02T13:18:20,325 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:20,325 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:20,325 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,326 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.2 K 2024-12-02T13:18:20,326 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1e2708b4bc49f3af6951f18710e833, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733145497945 2024-12-02T13:18:20,326 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c0762efa64fc4549aeb643d2c479ceeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733145498044 2024-12-02T13:18:20,327 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b174ecdd564c33a63af75f451b4017, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:20,335 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:20,336 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0d273c74ed134bf4bf2415e77c9d59e3 is 50, key is test_row_0/C:col10/1733145499148/Put/seqid=0 2024-12-02T13:18:20,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742080_1256 (size=12104) 2024-12-02T13:18:20,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,392 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=117.74 KB heapSize=333.55 KB 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:20,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/15dfbf9016b2441f96f9635a0e81d13b is 50, key is test_row_0/A:col10/1733145499883/Put/seqid=0 2024-12-02T13:18:20,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742081_1257 (size=12001) 2024-12-02T13:18:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T13:18:20,405 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/15dfbf9016b2441f96f9635a0e81d13b 2024-12-02T13:18:20,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/05510bbf66f94b6a94a615f72c6251d2 is 50, key is test_row_0/B:col10/1733145499883/Put/seqid=0 2024-12-02T13:18:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742082_1258 (size=12001) 2024-12-02T13:18:20,429 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/05510bbf66f94b6a94a615f72c6251d2 2024-12-02T13:18:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6ef0cc897c3a4e0884251e5e11c730d2 is 50, key is test_row_0/C:col10/1733145499883/Put/seqid=0 2024-12-02T13:18:20,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742083_1259 (size=12001) 2024-12-02T13:18:20,480 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6ef0cc897c3a4e0884251e5e11c730d2 2024-12-02T13:18:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/15dfbf9016b2441f96f9635a0e81d13b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b 2024-12-02T13:18:20,498 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T13:18:20,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/05510bbf66f94b6a94a615f72c6251d2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2 2024-12-02T13:18:20,506 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T13:18:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6ef0cc897c3a4e0884251e5e11c730d2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2 2024-12-02T13:18:20,518 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2, entries=150, sequenceid=96, filesize=11.7 K 2024-12-02T13:18:20,519 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~117.74 KB/120562, heapSize ~333.50 KB/341504, currentSize=0 B/0 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 127ms, sequenceid=96, compaction requested=false 2024-12-02T13:18:20,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:20,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T13:18:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T13:18:20,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T13:18:20,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 434 msec 2024-12-02T13:18:20,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 440 msec 2024-12-02T13:18:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:20,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:20,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:20,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ff8bf2027e3f45b1a34fcbccf7b61961 is 50, key is test_row_0/A:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:20,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742084_1260 (size=16681) 2024-12-02T13:18:20,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ff8bf2027e3f45b1a34fcbccf7b61961 2024-12-02T13:18:20,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/611b1f7566d84cf5be9ae66f01c9ac3c is 50, key is test_row_0/B:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742085_1261 (size=12001) 2024-12-02T13:18:20,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/611b1f7566d84cf5be9ae66f01c9ac3c 2024-12-02T13:18:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145560586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145500579, max timestamp=1733145500588Num uniques 859; 2024-12-02T13:18:20,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:20,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145500579, max timestamp=1733145500588Num uniques 859; 2024-12-02T13:18:20,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145500579, max timestamp=1733145500588Num uniques 859; 2024-12-02T13:18:20,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:20,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:20,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 is 50, key is test_row_0/C:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:20,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742086_1262 (size=12001) 2024-12-02T13:18:20,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145560636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145560639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145560694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T13:18:20,714 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:20,714 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T13:18:20,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:20,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T13:18:20,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:20,734 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f37e10de16bf49b99a4d519b09959e0f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f37e10de16bf49b99a4d519b09959e0f 2024-12-02T13:18:20,761 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into f37e10de16bf49b99a4d519b09959e0f(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:20,761 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:20,761 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145500277; duration=0sec 2024-12-02T13:18:20,761 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:20,761 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145560757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145560757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,764 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0d273c74ed134bf4bf2415e77c9d59e3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0d273c74ed134bf4bf2415e77c9d59e3 2024-12-02T13:18:20,768 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 0d273c74ed134bf4bf2415e77c9d59e3(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:20,768 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:20,768 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145500277; duration=0sec 2024-12-02T13:18:20,768 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:20,768 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T13:18:20,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T13:18:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:20,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:20,889 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:18:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145560904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145560973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145560973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:20,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 2024-12-02T13:18:21,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T13:18:21,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:21,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:21,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ff8bf2027e3f45b1a34fcbccf7b61961 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961 2024-12-02T13:18:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:21,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961, entries=250, sequenceid=107, filesize=16.3 K 2024-12-02T13:18:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T13:18:21,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/611b1f7566d84cf5be9ae66f01c9ac3c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c 2024-12-02T13:18:21,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c, entries=150, sequenceid=107, filesize=11.7 K 2024-12-02T13:18:21,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 2024-12-02T13:18:21,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3, entries=150, sequenceid=107, filesize=11.7 K 2024-12-02T13:18:21,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 505ms, sequenceid=107, compaction requested=true 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:21,048 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:21,048 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:21,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:21,049 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:21,049 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:21,049 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:21,049 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:21,049 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,049 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,049 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.3 K 2024-12-02T13:18:21,049 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f37e10de16bf49b99a4d519b09959e0f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=39.8 K 2024-12-02T13:18:21,057 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c49f0a6c79ac4c8485b2c3c3fe4d4a2f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:21,057 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f37e10de16bf49b99a4d519b09959e0f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:21,058 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 05510bbf66f94b6a94a615f72c6251d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733145499883 2024-12-02T13:18:21,059 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15dfbf9016b2441f96f9635a0e81d13b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733145499883 2024-12-02T13:18:21,059 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 611b1f7566d84cf5be9ae66f01c9ac3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:21,059 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff8bf2027e3f45b1a34fcbccf7b61961, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:21,077 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#213 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:21,078 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b5468e2d63694e67bab9a3fbc3d2b97a is 50, key is test_row_0/B:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:21,083 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:21,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/cbd5a3a85c0e4c1493e35a301e4e6692 is 50, key is test_row_0/A:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:21,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742087_1263 (size=12207) 2024-12-02T13:18:21,118 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b5468e2d63694e67bab9a3fbc3d2b97a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b5468e2d63694e67bab9a3fbc3d2b97a 2024-12-02T13:18:21,124 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into b5468e2d63694e67bab9a3fbc3d2b97a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:21,124 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,124 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145501048; duration=0sec 2024-12-02T13:18:21,124 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:21,124 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:21,125 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:21,126 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:21,126 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:21,126 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,126 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0d273c74ed134bf4bf2415e77c9d59e3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.3 K 2024-12-02T13:18:21,126 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d273c74ed134bf4bf2415e77c9d59e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145499148 2024-12-02T13:18:21,127 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef0cc897c3a4e0884251e5e11c730d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733145499883 2024-12-02T13:18:21,127 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b38fa2a8ff4fdcb4b8f947de45d7f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:21,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742088_1264 (size=12207) 2024-12-02T13:18:21,143 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:21,144 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/51d5080c8043407cbace620b6fb728a5 is 50, key is test_row_0/C:col10/1733145500535/Put/seqid=0 2024-12-02T13:18:21,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742089_1265 (size=12207) 2024-12-02T13:18:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T13:18:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,185 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:21,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a0064a1e1cef4336bb0beda8817e2834 is 50, key is test_row_0/A:col10/1733145500634/Put/seqid=0 2024-12-02T13:18:21,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742090_1266 (size=12151) 2024-12-02T13:18:21,205 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a0064a1e1cef4336bb0beda8817e2834 2024-12-02T13:18:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ee9fd0ba8bb84a02ad440dddbcb5b775 is 50, key is test_row_0/B:col10/1733145500634/Put/seqid=0 2024-12-02T13:18:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:21,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:21,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742091_1267 (size=12151) 2024-12-02T13:18:21,263 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ee9fd0ba8bb84a02ad440dddbcb5b775 2024-12-02T13:18:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145561267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145561284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/355fceb69b6243e0822518046e3a470e is 50, key is test_row_0/C:col10/1733145500634/Put/seqid=0 2024-12-02T13:18:21,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145561284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742092_1268 (size=12151) 2024-12-02T13:18:21,343 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/355fceb69b6243e0822518046e3a470e 2024-12-02T13:18:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T13:18:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a0064a1e1cef4336bb0beda8817e2834 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834 2024-12-02T13:18:21,358 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T13:18:21,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ee9fd0ba8bb84a02ad440dddbcb5b775 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775 2024-12-02T13:18:21,364 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T13:18:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/355fceb69b6243e0822518046e3a470e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e 2024-12-02T13:18:21,375 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T13:18:21,377 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 193ms, sequenceid=145, compaction requested=false 2024-12-02T13:18:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T13:18:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T13:18:21,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T13:18:21,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 660 msec 2024-12-02T13:18:21,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:21,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:21,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 673 msec 2024-12-02T13:18:21,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9798c999c59243c29f78a5511567c732 is 50, key is test_row_0/A:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:21,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742093_1269 (size=12151) 2024-12-02T13:18:21,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9798c999c59243c29f78a5511567c732 2024-12-02T13:18:21,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d6fecc203880451582c4c5968ba947d7 is 50, key is test_row_0/B:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:21,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742094_1270 (size=12151) 2024-12-02T13:18:21,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d6fecc203880451582c4c5968ba947d7 2024-12-02T13:18:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145561494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9f989250de343e7b425c619a7d71cfd is 50, key is test_row_0/C:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:21,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742095_1271 (size=12151) 2024-12-02T13:18:21,532 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/cbd5a3a85c0e4c1493e35a301e4e6692 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cbd5a3a85c0e4c1493e35a301e4e6692 2024-12-02T13:18:21,545 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into cbd5a3a85c0e4c1493e35a301e4e6692(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:21,545 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,545 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145501048; duration=0sec 2024-12-02T13:18:21,545 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:21,545 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:21,577 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/51d5080c8043407cbace620b6fb728a5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/51d5080c8043407cbace620b6fb728a5 2024-12-02T13:18:21,586 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 51d5080c8043407cbace620b6fb728a5(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:21,586 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,586 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145501048; duration=0sec 2024-12-02T13:18:21,587 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:21,587 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145561604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145561794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145561794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145561814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:21,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T13:18:21,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:21,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T13:18:21,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T13:18:21,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:21,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:21,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9f989250de343e7b425c619a7d71cfd 2024-12-02T13:18:21,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9798c999c59243c29f78a5511567c732 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732 2024-12-02T13:18:21,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732, entries=150, sequenceid=157, filesize=11.9 K 2024-12-02T13:18:21,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d6fecc203880451582c4c5968ba947d7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7 2024-12-02T13:18:21,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7, entries=150, sequenceid=157, filesize=11.9 K 2024-12-02T13:18:21,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9f989250de343e7b425c619a7d71cfd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd 2024-12-02T13:18:21,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd, entries=150, sequenceid=157, filesize=11.9 K 2024-12-02T13:18:21,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 574ms, sequenceid=157, compaction requested=true 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:21,963 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:21,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:18:21,963 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T13:18:21,964 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:21,964 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:21,964 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:21,964 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:21,964 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,965 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:21,965 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b5468e2d63694e67bab9a3fbc3d2b97a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.7 K 2024-12-02T13:18:21,965 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cbd5a3a85c0e4c1493e35a301e4e6692, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.7 K 2024-12-02T13:18:21,965 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b5468e2d63694e67bab9a3fbc3d2b97a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:21,965 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd5a3a85c0e4c1493e35a301e4e6692, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:21,965 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9fd0ba8bb84a02ad440dddbcb5b775, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733145500626 2024-12-02T13:18:21,966 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0064a1e1cef4336bb0beda8817e2834, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733145500626 2024-12-02T13:18:21,966 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9798c999c59243c29f78a5511567c732, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:21,966 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d6fecc203880451582c4c5968ba947d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:21,983 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:21,983 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/df8dbe11cdd64b0c8b57b859497fe5c1 is 50, key is test_row_0/A:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:21,994 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:21,994 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/02bb90be0d3347fc82e749c6fe942e94 is 50, key is test_row_0/B:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:22,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T13:18:22,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:22,015 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742096_1272 (size=12459) 2024-12-02T13:18:22,025 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/df8dbe11cdd64b0c8b57b859497fe5c1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/df8dbe11cdd64b0c8b57b859497fe5c1 2024-12-02T13:18:22,031 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into df8dbe11cdd64b0c8b57b859497fe5c1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:22,031 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:22,031 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145501963; duration=0sec 2024-12-02T13:18:22,031 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:22,031 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:22,031 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:22,033 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:22,033 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:22,033 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:22,033 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/51d5080c8043407cbace620b6fb728a5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=35.7 K 2024-12-02T13:18:22,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/31e7bd1a749c4a3cbd14e5a27e39f62e is 50, key is test_row_0/A:col10/1733145501492/Put/seqid=0 2024-12-02T13:18:22,036 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d5080c8043407cbace620b6fb728a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733145500535 2024-12-02T13:18:22,036 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355fceb69b6243e0822518046e3a470e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733145500626 2024-12-02T13:18:22,037 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f989250de343e7b425c619a7d71cfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:22,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742097_1273 (size=12459) 2024-12-02T13:18:22,076 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:22,076 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/e567ae9397234feaa67032332ceae94a is 50, key is test_row_0/C:col10/1733145501250/Put/seqid=0 2024-12-02T13:18:22,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742098_1274 (size=12151) 2024-12-02T13:18:22,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742099_1275 (size=12459) 2024-12-02T13:18:22,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:22,132 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/e567ae9397234feaa67032332ceae94a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/e567ae9397234feaa67032332ceae94a 2024-12-02T13:18:22,138 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into e567ae9397234feaa67032332ceae94a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:22,138 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:22,138 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145501963; duration=0sec 2024-12-02T13:18:22,139 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:22,139 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T13:18:22,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145562173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145562224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,227 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145562244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,248 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145562284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,466 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/02bb90be0d3347fc82e749c6fe942e94 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/02bb90be0d3347fc82e749c6fe942e94 2024-12-02T13:18:22,470 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into 02bb90be0d3347fc82e749c6fe942e94(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:22,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:22,470 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145501963; duration=0sec 2024-12-02T13:18:22,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:22,470 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T13:18:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145562493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,503 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/31e7bd1a749c4a3cbd14e5a27e39f62e 2024-12-02T13:18:22,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/456cad7b634c4e9fa329e6855ae8b056 is 50, key is test_row_0/B:col10/1733145501492/Put/seqid=0 2024-12-02T13:18:22,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742100_1276 (size=12151) 2024-12-02T13:18:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145562804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145562804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145562804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:22,805 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,806 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:22,913 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/456cad7b634c4e9fa329e6855ae8b056 2024-12-02T13:18:22,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9d8395ec0044dd38529de4dc1226d3e is 50, key is test_row_0/C:col10/1733145501492/Put/seqid=0 2024-12-02T13:18:22,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742101_1277 (size=12151) 2024-12-02T13:18:22,931 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9d8395ec0044dd38529de4dc1226d3e 2024-12-02T13:18:22,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/31e7bd1a749c4a3cbd14e5a27e39f62e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e 2024-12-02T13:18:22,942 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T13:18:22,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/456cad7b634c4e9fa329e6855ae8b056 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056 2024-12-02T13:18:22,949 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T13:18:22,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/f9d8395ec0044dd38529de4dc1226d3e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e 2024-12-02T13:18:22,958 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T13:18:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-02T13:18:22,959 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 944ms, sequenceid=179, compaction requested=false 2024-12-02T13:18:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:22,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T13:18:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T13:18:22,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T13:18:22,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1020 sec 2024-12-02T13:18:22,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.1080 sec 2024-12-02T13:18:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T13:18:22,994 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:22,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T13:18:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T13:18:22,997 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:22,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:22,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T13:18:23,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T13:18:23,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:23,150 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:23,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f077398ba3614315bf12c441ba270663 is 50, key is test_row_0/A:col10/1733145502170/Put/seqid=0 2024-12-02T13:18:23,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742102_1278 (size=12151) 2024-12-02T13:18:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T13:18:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:23,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:23,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145563407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:23,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145563514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:23,595 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f077398ba3614315bf12c441ba270663 2024-12-02T13:18:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/774bc102ba0e40c4a10925f79573c974 is 50, key is test_row_0/B:col10/1733145502170/Put/seqid=0 2024-12-02T13:18:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T13:18:23,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742103_1279 (size=12151) 2024-12-02T13:18:23,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145563723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,026 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/774bc102ba0e40c4a10925f79573c974 2024-12-02T13:18:24,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/86efbde0036f4d189522d5b2f4e4c551 is 50, key is test_row_0/C:col10/1733145502170/Put/seqid=0 2024-12-02T13:18:24,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145564034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742104_1280 (size=12151) 2024-12-02T13:18:24,048 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/86efbde0036f4d189522d5b2f4e4c551 2024-12-02T13:18:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f077398ba3614315bf12c441ba270663 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663 2024-12-02T13:18:24,058 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T13:18:24,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/774bc102ba0e40c4a10925f79573c974 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974 2024-12-02T13:18:24,064 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T13:18:24,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/86efbde0036f4d189522d5b2f4e4c551 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551 2024-12-02T13:18:24,069 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T13:18:24,069 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 919ms, sequenceid=194, compaction requested=true 2024-12-02T13:18:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:24,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T13:18:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T13:18:24,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T13:18:24,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-12-02T13:18:24,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0790 sec 2024-12-02T13:18:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T13:18:24,134 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:24,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T13:18:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T13:18:24,137 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:24,138 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:24,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T13:18:24,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T13:18:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:24,291 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:18:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 is 50, key is test_row_0/A:col10/1733145503400/Put/seqid=0 2024-12-02T13:18:24,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742105_1281 (size=12151) 2024-12-02T13:18:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T13:18:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:24,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:24,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145564576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145564683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,711 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 2024-12-02T13:18:24,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/0ae381403edd4695bcca8210f5fa7e62 is 50, key is test_row_0/B:col10/1733145503400/Put/seqid=0 2024-12-02T13:18:24,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742106_1282 (size=12151) 2024-12-02T13:18:24,740 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/0ae381403edd4695bcca8210f5fa7e62 2024-12-02T13:18:24,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/7922cb80885040159e89beb04205a0dd is 50, key is test_row_0/C:col10/1733145503400/Put/seqid=0 2024-12-02T13:18:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T13:18:24,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742107_1283 (size=12151) 2024-12-02T13:18:24,767 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/7922cb80885040159e89beb04205a0dd 2024-12-02T13:18:24,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 2024-12-02T13:18:24,779 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T13:18:24,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/0ae381403edd4695bcca8210f5fa7e62 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62 2024-12-02T13:18:24,789 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T13:18:24,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/7922cb80885040159e89beb04205a0dd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd 2024-12-02T13:18:24,793 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd, entries=150, sequenceid=214, filesize=11.9 K 2024-12-02T13:18:24,794 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 503ms, sequenceid=214, compaction requested=true 2024-12-02T13:18:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T13:18:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T13:18:24,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T13:18:24,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 660 msec 2024-12-02T13:18:24,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 669 msec 2024-12-02T13:18:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:24,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:24,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:24,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5b16a59de92b4bb980407ea547d437d3 is 50, key is test_row_0/A:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:24,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742108_1284 (size=14541) 2024-12-02T13:18:24,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5b16a59de92b4bb980407ea547d437d3 2024-12-02T13:18:24,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/228ee67c25644b5aba32edee25767391 is 50, key is test_row_0/B:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:24,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742109_1285 (size=12151) 2024-12-02T13:18:24,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145564871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145564875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145564893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145564984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145564984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145565194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145565194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145565204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T13:18:25,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/228ee67c25644b5aba32edee25767391 2024-12-02T13:18:25,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:25,275 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-02T13:18:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:25,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:25,278 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:25,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:25,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/535177b893f948de9f9b9780c7622219 is 50, key is test_row_0/C:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:25,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742110_1286 (size=12151) 2024-12-02T13:18:25,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/535177b893f948de9f9b9780c7622219 2024-12-02T13:18:25,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5b16a59de92b4bb980407ea547d437d3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3 2024-12-02T13:18:25,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3, entries=200, sequenceid=227, filesize=14.2 K 2024-12-02T13:18:25,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/228ee67c25644b5aba32edee25767391 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391 2024-12-02T13:18:25,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T13:18:25,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/535177b893f948de9f9b9780c7622219 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219 2024-12-02T13:18:25,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219, entries=150, sequenceid=227, filesize=11.9 K 2024-12-02T13:18:25,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 533ms, sequenceid=227, compaction requested=true 2024-12-02T13:18:25,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:25,349 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:25,349 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:25,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:25,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:18:25,355 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63453 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:18:25,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:25,355 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:25,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:25,355 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:25,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/02bb90be0d3347fc82e749c6fe942e94, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=59.6 K 2024-12-02T13:18:25,355 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/df8dbe11cdd64b0c8b57b859497fe5c1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=62.0 K 2024-12-02T13:18:25,357 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 02bb90be0d3347fc82e749c6fe942e94, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:25,357 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting df8dbe11cdd64b0c8b57b859497fe5c1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:25,358 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31e7bd1a749c4a3cbd14e5a27e39f62e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145501467 2024-12-02T13:18:25,358 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 456cad7b634c4e9fa329e6855ae8b056, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145501467 2024-12-02T13:18:25,358 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 774bc102ba0e40c4a10925f79573c974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733145502137 2024-12-02T13:18:25,358 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f077398ba3614315bf12c441ba270663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733145502137 2024-12-02T13:18:25,360 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f67ccde1ee4cf0917815fa1ca4d7c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733145503394 2024-12-02T13:18:25,360 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae381403edd4695bcca8210f5fa7e62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733145503394 2024-12-02T13:18:25,361 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b16a59de92b4bb980407ea547d437d3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:25,361 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 228ee67c25644b5aba32edee25767391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:25,381 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:25,382 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9aa6dc4c734448f1bf20b4619f904440 is 50, key is test_row_0/A:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:25,396 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#238 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:25,396 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/790a53bb99c446f4a3953c74b987679b is 50, key is test_row_0/B:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742111_1287 (size=12629) 2024-12-02T13:18:25,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T13:18:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:25,432 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:25,438 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9aa6dc4c734448f1bf20b4619f904440 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9aa6dc4c734448f1bf20b4619f904440 2024-12-02T13:18:25,444 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into 9aa6dc4c734448f1bf20b4619f904440(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:25,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:25,444 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=11, startTime=1733145505349; duration=0sec 2024-12-02T13:18:25,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:25,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:25,444 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T13:18:25,445 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T13:18:25,445 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:25,445 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:25,446 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/e567ae9397234feaa67032332ceae94a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=59.6 K 2024-12-02T13:18:25,447 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e567ae9397234feaa67032332ceae94a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733145501250 2024-12-02T13:18:25,448 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d8395ec0044dd38529de4dc1226d3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145501467 2024-12-02T13:18:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/0e6d1c0a80834cacad320697a6582599 is 50, key is test_row_0/A:col10/1733145504861/Put/seqid=0 2024-12-02T13:18:25,449 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86efbde0036f4d189522d5b2f4e4c551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733145502137 2024-12-02T13:18:25,454 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7922cb80885040159e89beb04205a0dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733145503394 2024-12-02T13:18:25,455 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 535177b893f948de9f9b9780c7622219, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:25,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742112_1288 (size=12629) 2024-12-02T13:18:25,468 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/790a53bb99c446f4a3953c74b987679b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/790a53bb99c446f4a3953c74b987679b 2024-12-02T13:18:25,476 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into 790a53bb99c446f4a3953c74b987679b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:25,476 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:25,476 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=11, startTime=1733145505349; duration=0sec 2024-12-02T13:18:25,476 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:25,476 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:25,485 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:25,485 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/97aad5750b314d00b5155e19e850cd0e is 50, key is test_row_0/C:col10/1733145504814/Put/seqid=0 2024-12-02T13:18:25,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742113_1289 (size=12151) 2024-12-02T13:18:25,495 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/0e6d1c0a80834cacad320697a6582599 2024-12-02T13:18:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:25,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:25,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742114_1290 (size=12629) 2024-12-02T13:18:25,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fa547d98a38a411c9e92216a89989251 is 50, key is test_row_0/B:col10/1733145504861/Put/seqid=0 2024-12-02T13:18:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145565528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145565529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,540 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/97aad5750b314d00b5155e19e850cd0e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/97aad5750b314d00b5155e19e850cd0e 2024-12-02T13:18:25,546 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 97aad5750b314d00b5155e19e850cd0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:25,546 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:25,546 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=11, startTime=1733145505349; duration=0sec 2024-12-02T13:18:25,546 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:25,546 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:25,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742115_1291 (size=12151) 2024-12-02T13:18:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145565644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145565644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145565714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145565854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145565855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:25,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:25,975 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fa547d98a38a411c9e92216a89989251 2024-12-02T13:18:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 is 50, key is test_row_0/C:col10/1733145504861/Put/seqid=0 2024-12-02T13:18:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742116_1292 (size=12151) 2024-12-02T13:18:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145566177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145566178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145566263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,266 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145566273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,275 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,411 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 2024-12-02T13:18:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/0e6d1c0a80834cacad320697a6582599 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599 2024-12-02T13:18:26,418 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599, entries=150, sequenceid=246, filesize=11.9 K 2024-12-02T13:18:26,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fa547d98a38a411c9e92216a89989251 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251 2024-12-02T13:18:26,422 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251, entries=150, sequenceid=246, filesize=11.9 K 2024-12-02T13:18:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 2024-12-02T13:18:26,427 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50, entries=150, sequenceid=246, filesize=11.9 K 2024-12-02T13:18:26,428 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 995ms, sequenceid=246, compaction requested=false 2024-12-02T13:18:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-02T13:18:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-02T13:18:26,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T13:18:26,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1510 sec 2024-12-02T13:18:26,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1560 sec 2024-12-02T13:18:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:26,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:26,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:26,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9c24543b6fa64a04928d1bfdb423e09b is 50, key is test_row_0/A:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:26,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742117_1293 (size=14691) 2024-12-02T13:18:26,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145566724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145566723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145566724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,729 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145566833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145566833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145567043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145567043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9c24543b6fa64a04928d1bfdb423e09b 2024-12-02T13:18:27,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/7a5f59ed2f084b8db7cb99e6a6e6581e is 50, key is test_row_0/B:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:27,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742118_1294 (size=12251) 2024-12-02T13:18:27,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/7a5f59ed2f084b8db7cb99e6a6e6581e 2024-12-02T13:18:27,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/df4ad470d3ea4b3eb80e406bd66d5dee is 50, key is test_row_0/C:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:27,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742119_1295 (size=12251) 2024-12-02T13:18:27,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/df4ad470d3ea4b3eb80e406bd66d5dee 2024-12-02T13:18:27,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/9c24543b6fa64a04928d1bfdb423e09b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b 2024-12-02T13:18:27,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b, entries=200, sequenceid=264, filesize=14.3 K 2024-12-02T13:18:27,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/7a5f59ed2f084b8db7cb99e6a6e6581e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e 2024-12-02T13:18:27,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e, entries=150, sequenceid=264, filesize=12.0 K 2024-12-02T13:18:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/df4ad470d3ea4b3eb80e406bd66d5dee as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee 2024-12-02T13:18:27,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee, entries=150, sequenceid=264, filesize=12.0 K 2024-12-02T13:18:27,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 508ms, sequenceid=264, compaction requested=true 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:27,194 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:27,194 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:27,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:27,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:27,197 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,197 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/790a53bb99c446f4a3953c74b987679b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.2 K 2024-12-02T13:18:27,197 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:27,197 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:27,197 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,197 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9aa6dc4c734448f1bf20b4619f904440, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=38.5 K 2024-12-02T13:18:27,197 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 790a53bb99c446f4a3953c74b987679b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:27,197 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aa6dc4c734448f1bf20b4619f904440, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:27,198 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e6d1c0a80834cacad320697a6582599, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145504861 2024-12-02T13:18:27,198 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fa547d98a38a411c9e92216a89989251, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145504861 2024-12-02T13:18:27,198 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5f59ed2f084b8db7cb99e6a6e6581e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:27,198 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c24543b6fa64a04928d1bfdb423e09b, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:27,207 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:27,208 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fceeb73c399f46808f7babf5ac748a32 is 50, key is test_row_0/B:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:27,211 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:27,211 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a4478757bf4d470381b49edf22876cb4 is 50, key is test_row_0/A:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:27,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742121_1297 (size=12831) 2024-12-02T13:18:27,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742120_1296 (size=12831) 2024-12-02T13:18:27,263 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a4478757bf4d470381b49edf22876cb4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a4478757bf4d470381b49edf22876cb4 2024-12-02T13:18:27,267 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fceeb73c399f46808f7babf5ac748a32 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fceeb73c399f46808f7babf5ac748a32 2024-12-02T13:18:27,268 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into a4478757bf4d470381b49edf22876cb4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:27,268 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:27,268 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145507194; duration=0sec 2024-12-02T13:18:27,268 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:27,268 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:27,268 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:27,270 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:27,270 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:27,270 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,270 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/97aad5750b314d00b5155e19e850cd0e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.2 K 2024-12-02T13:18:27,270 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97aad5750b314d00b5155e19e850cd0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1733145504559 2024-12-02T13:18:27,271 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24e2f4ee717a4ad2bc67c19d0e7d7b50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733145504861 2024-12-02T13:18:27,271 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4ad470d3ea4b3eb80e406bd66d5dee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:27,273 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into fceeb73c399f46808f7babf5ac748a32(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:27,273 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:27,273 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145507194; duration=0sec 2024-12-02T13:18:27,274 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:27,274 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:27,279 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:27,280 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/092c1596772742e4b61866bcda1f21a6 is 50, key is test_row_0/C:col10/1733145505515/Put/seqid=0 2024-12-02T13:18:27,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742122_1298 (size=12831) 2024-12-02T13:18:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:27,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:27,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e is 50, key is test_row_0/A:col10/1733145507357/Put/seqid=0 2024-12-02T13:18:27,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742123_1299 (size=14741) 2024-12-02T13:18:27,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e 2024-12-02T13:18:27,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145567385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145567387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/61166c296d3a42a99768e80648dcbb9b is 50, key is test_row_0/B:col10/1733145507357/Put/seqid=0 2024-12-02T13:18:27,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742124_1300 (size=12301) 2024-12-02T13:18:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T13:18:27,424 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:27,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-02T13:18:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:27,426 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:27,427 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:27,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:27,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145567494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145567497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:27,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T13:18:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,688 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/092c1596772742e4b61866bcda1f21a6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/092c1596772742e4b61866bcda1f21a6 2024-12-02T13:18:27,695 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 092c1596772742e4b61866bcda1f21a6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:27,695 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:27,695 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145507194; duration=0sec 2024-12-02T13:18:27,695 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:27,695 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145567704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145567704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:27,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T13:18:27,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:27,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/61166c296d3a42a99768e80648dcbb9b 2024-12-02T13:18:27,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/3f0c2551a728497eacc7509e6da2d165 is 50, key is test_row_0/C:col10/1733145507357/Put/seqid=0 2024-12-02T13:18:27,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742125_1301 (size=12301) 2024-12-02T13:18:27,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/3f0c2551a728497eacc7509e6da2d165 2024-12-02T13:18:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e 2024-12-02T13:18:27,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e, entries=200, sequenceid=285, filesize=14.4 K 2024-12-02T13:18:27,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/61166c296d3a42a99768e80648dcbb9b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b 2024-12-02T13:18:27,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b, entries=150, sequenceid=285, filesize=12.0 K 2024-12-02T13:18:27,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/3f0c2551a728497eacc7509e6da2d165 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165 2024-12-02T13:18:27,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165, entries=150, sequenceid=285, filesize=12.0 K 2024-12-02T13:18:27,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 502ms, sequenceid=285, compaction requested=false 2024-12-02T13:18:27,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:27,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:27,888 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d19ab4b32b5b4a1d8334b7d89b7d671b is 50, key is test_row_0/A:col10/1733145507382/Put/seqid=0 2024-12-02T13:18:27,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742126_1302 (size=12301) 2024-12-02T13:18:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:28,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:28,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145568048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:28,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145568053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145568164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145568164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,321 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d19ab4b32b5b4a1d8334b7d89b7d671b 2024-12-02T13:18:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f9a02b3211c946c597df99e78a282be4 is 50, key is test_row_0/B:col10/1733145507382/Put/seqid=0 2024-12-02T13:18:28,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742127_1303 (size=12301) 2024-12-02T13:18:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145568374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145568374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:28,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145568684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145568684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,734 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f9a02b3211c946c597df99e78a282be4 2024-12-02T13:18:28,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145568743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:28,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/ed13eed6a53b41f1b76abe4c546cb976 is 50, key is test_row_0/C:col10/1733145507382/Put/seqid=0 2024-12-02T13:18:28,749 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742128_1304 (size=12301) 2024-12-02T13:18:29,172 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/ed13eed6a53b41f1b76abe4c546cb976 2024-12-02T13:18:29,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d19ab4b32b5b4a1d8334b7d89b7d671b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b 2024-12-02T13:18:29,184 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b, entries=150, sequenceid=299, filesize=12.0 K 2024-12-02T13:18:29,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f9a02b3211c946c597df99e78a282be4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4 2024-12-02T13:18:29,190 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4, entries=150, sequenceid=299, filesize=12.0 K 2024-12-02T13:18:29,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/ed13eed6a53b41f1b76abe4c546cb976 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976 2024-12-02T13:18:29,195 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976, entries=150, sequenceid=299, filesize=12.0 K 2024-12-02T13:18:29,197 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1308ms, sequenceid=299, compaction requested=true 2024-12-02T13:18:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-02T13:18:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-02T13:18:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:29,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:29,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:29,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T13:18:29,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-12-02T13:18:29,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7760 sec 2024-12-02T13:18:29,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d3a7a844d4964516bf47cf82642f97fa is 50, key is test_row_0/A:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:29,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742129_1305 (size=14741) 2024-12-02T13:18:29,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d3a7a844d4964516bf47cf82642f97fa 2024-12-02T13:18:29,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 is 50, key is test_row_0/B:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145569225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145569226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742130_1306 (size=12301) 2024-12-02T13:18:29,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145569334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145569335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145569544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145569544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T13:18:29,574 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:29,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-02T13:18:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:29,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:29,576 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:29,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:29,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 2024-12-02T13:18:29,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c47a7ee66d4c4f178d5fc00b63e24b8c is 50, key is test_row_0/C:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:29,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742131_1307 (size=12301) 2024-12-02T13:18:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:29,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T13:18:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:29,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145569854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145569854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:29,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T13:18:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:29,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:30,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T13:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:30,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:30,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c47a7ee66d4c4f178d5fc00b63e24b8c 2024-12-02T13:18:30,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d3a7a844d4964516bf47cf82642f97fa as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa 2024-12-02T13:18:30,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa, entries=200, sequenceid=319, filesize=14.4 K 2024-12-02T13:18:30,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 2024-12-02T13:18:30,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T13:18:30,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c47a7ee66d4c4f178d5fc00b63e24b8c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c 2024-12-02T13:18:30,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T13:18:30,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 896ms, sequenceid=319, compaction requested=true 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:30,093 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:30,093 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:30,095 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54614 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:30,095 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:30,095 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,095 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:30,095 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:30,095 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a4478757bf4d470381b49edf22876cb4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=53.3 K 2024-12-02T13:18:30,095 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,095 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fceeb73c399f46808f7babf5ac748a32, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=48.6 K 2024-12-02T13:18:30,096 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4478757bf4d470381b49edf22876cb4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:30,096 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1c0d07cb5d4ed2846cd12ca0e0c29e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733145506722 2024-12-02T13:18:30,096 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fceeb73c399f46808f7babf5ac748a32, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:30,096 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19ab4b32b5b4a1d8334b7d89b7d671b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733145507367 2024-12-02T13:18:30,096 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 61166c296d3a42a99768e80648dcbb9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733145506722 2024-12-02T13:18:30,097 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a02b3211c946c597df99e78a282be4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733145507367 2024-12-02T13:18:30,097 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a7a844d4964516bf47cf82642f97fa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:30,097 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b3fa64ba3f0f474d8663f2d6f1c7c9d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:30,112 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:30,112 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:30,112 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/0bd3479fe7ac4e9ca717b256c2645891 is 50, key is test_row_0/A:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:30,112 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/74bd7cab83104112910da1e100c1320a is 50, key is test_row_0/B:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742132_1308 (size=13017) 2024-12-02T13:18:30,132 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/74bd7cab83104112910da1e100c1320a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/74bd7cab83104112910da1e100c1320a 2024-12-02T13:18:30,139 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into 74bd7cab83104112910da1e100c1320a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:30,139 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:30,139 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=12, startTime=1733145510093; duration=0sec 2024-12-02T13:18:30,139 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:30,139 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:30,139 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:30,141 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:30,141 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:30,141 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,142 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/092c1596772742e4b61866bcda1f21a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=48.6 K 2024-12-02T13:18:30,142 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 092c1596772742e4b61866bcda1f21a6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733145505515 2024-12-02T13:18:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742133_1309 (size=13017) 2024-12-02T13:18:30,142 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0c2551a728497eacc7509e6da2d165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733145506722 2024-12-02T13:18:30,143 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ed13eed6a53b41f1b76abe4c546cb976, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733145507367 2024-12-02T13:18:30,143 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c47a7ee66d4c4f178d5fc00b63e24b8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:30,151 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/0bd3479fe7ac4e9ca717b256c2645891 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0bd3479fe7ac4e9ca717b256c2645891 2024-12-02T13:18:30,158 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into 0bd3479fe7ac4e9ca717b256c2645891(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:30,160 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:30,160 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=12, startTime=1733145510093; duration=0sec 2024-12-02T13:18:30,160 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:30,160 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:30,178 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:30,179 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/fe710033d37841288100ec51fd332446 is 50, key is test_row_0/C:col10/1733145509196/Put/seqid=0 2024-12-02T13:18:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742134_1310 (size=13017) 2024-12-02T13:18:30,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T13:18:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:30,189 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:30,192 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/fe710033d37841288100ec51fd332446 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/fe710033d37841288100ec51fd332446 2024-12-02T13:18:30,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f5ae102175d84d2fbff37f324a6f5b20 is 50, key is test_row_0/A:col10/1733145509223/Put/seqid=0 2024-12-02T13:18:30,198 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into fe710033d37841288100ec51fd332446(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:30,198 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:30,198 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=12, startTime=1733145510093; duration=0sec 2024-12-02T13:18:30,198 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:30,199 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:30,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742135_1311 (size=12301) 2024-12-02T13:18:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:30,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145570390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:30,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145570391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:30,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145570503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145570503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:30,600 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f5ae102175d84d2fbff37f324a6f5b20 2024-12-02T13:18:30,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ec42c864c90f44b69f321d08af7df1ec is 50, key is test_row_0/B:col10/1733145509223/Put/seqid=0 2024-12-02T13:18:30,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742136_1312 (size=12301) 2024-12-02T13:18:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145570714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145570714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,012 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ec42c864c90f44b69f321d08af7df1ec 2024-12-02T13:18:31,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/66dc5858f5234af782dd836c62de0957 is 50, key is test_row_0/C:col10/1733145509223/Put/seqid=0 2024-12-02T13:18:31,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742137_1313 (size=12301) 2024-12-02T13:18:31,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145571023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145571024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,423 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/66dc5858f5234af782dd836c62de0957 2024-12-02T13:18:31,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f5ae102175d84d2fbff37f324a6f5b20 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20 2024-12-02T13:18:31,430 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T13:18:31,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/ec42c864c90f44b69f321d08af7df1ec as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec 2024-12-02T13:18:31,434 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T13:18:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/66dc5858f5234af782dd836c62de0957 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957 2024-12-02T13:18:31,439 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957, entries=150, sequenceid=333, filesize=12.0 K 2024-12-02T13:18:31,439 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1250ms, sequenceid=333, compaction requested=false 2024-12-02T13:18:31,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:31,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-02T13:18:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-02T13:18:31,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-02T13:18:31,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-12-02T13:18:31,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.8680 sec 2024-12-02T13:18:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:31,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:31,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:31,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b59acd367ea24973b54de8f1495686ee is 50, key is test_row_0/A:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:31,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742138_1314 (size=17181) 2024-12-02T13:18:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145571550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145571549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145571663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145571663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T13:18:31,724 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:31,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-12-02T13:18:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:31,726 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:31,727 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:31,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145571873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145571873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:31,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:31,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-02T13:18:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:31,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:31,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:31,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b59acd367ea24973b54de8f1495686ee 2024-12-02T13:18:31,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/1bf04cce630a4872808d591f6661caa1 is 50, key is test_row_0/B:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:31,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742139_1315 (size=12301) 2024-12-02T13:18:32,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-02T13:18:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:32,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-02T13:18:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145572184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145572184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-02T13:18:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:32,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/1bf04cce630a4872808d591f6661caa1 2024-12-02T13:18:32,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 is 50, key is test_row_0/C:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:32,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742140_1316 (size=12301) 2024-12-02T13:18:32,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 2024-12-02T13:18:32,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/b59acd367ea24973b54de8f1495686ee as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee 2024-12-02T13:18:32,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee, entries=250, sequenceid=356, filesize=16.8 K 2024-12-02T13:18:32,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/1bf04cce630a4872808d591f6661caa1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1 2024-12-02T13:18:32,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1, entries=150, sequenceid=356, filesize=12.0 K 2024-12-02T13:18:32,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 2024-12-02T13:18:32,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0, entries=150, sequenceid=356, filesize=12.0 K 2024-12-02T13:18:32,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 849ms, sequenceid=356, compaction requested=true 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:32,386 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:32,386 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:32,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:32,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:32,387 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:32,387 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:32,387 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:32,387 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:32,387 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,387 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,387 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/74bd7cab83104112910da1e100c1320a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.7 K 2024-12-02T13:18:32,387 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0bd3479fe7ac4e9ca717b256c2645891, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=41.5 K 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd3479fe7ac4e9ca717b256c2645891, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 74bd7cab83104112910da1e100c1320a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ae102175d84d2fbff37f324a6f5b20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733145509209 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ec42c864c90f44b69f321d08af7df1ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733145509209 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59acd367ea24973b54de8f1495686ee, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510378 2024-12-02T13:18:32,388 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf04cce630a4872808d591f6661caa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510384 2024-12-02T13:18:32,395 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#267 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:32,395 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:32,395 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/15992ea7775244e0bf72c200ef89ba0c is 50, key is test_row_0/A:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:32,395 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b35fcda9244d4b41842cd009c22841f3 is 50, key is test_row_0/B:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:32,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742142_1318 (size=13119) 2024-12-02T13:18:32,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742141_1317 (size=13119) 2024-12-02T13:18:32,408 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/15992ea7775244e0bf72c200ef89ba0c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15992ea7775244e0bf72c200ef89ba0c 2024-12-02T13:18:32,413 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b35fcda9244d4b41842cd009c22841f3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b35fcda9244d4b41842cd009c22841f3 2024-12-02T13:18:32,413 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into 15992ea7775244e0bf72c200ef89ba0c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:32,413 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:32,413 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145512386; duration=0sec 2024-12-02T13:18:32,413 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:32,413 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:32,414 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:32,415 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:32,415 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:32,415 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,415 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/fe710033d37841288100ec51fd332446, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.7 K 2024-12-02T13:18:32,416 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe710033d37841288100ec51fd332446, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733145508045 2024-12-02T13:18:32,416 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66dc5858f5234af782dd836c62de0957, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733145509209 2024-12-02T13:18:32,417 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c89d2c4a8d4bea8dd1fbd0a26d1cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510384 2024-12-02T13:18:32,418 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into b35fcda9244d4b41842cd009c22841f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:32,418 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:32,418 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145512386; duration=0sec 2024-12-02T13:18:32,418 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:32,418 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:32,423 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:32,424 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c102adff57894cf181c13b2cc88ccba7 is 50, key is test_row_0/C:col10/1733145510390/Put/seqid=0 2024-12-02T13:18:32,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742143_1319 (size=13119) 2024-12-02T13:18:32,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:32,490 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:32,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:32,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f60a8e843ce54bb29fffe54db518f025 is 50, key is test_row_0/A:col10/1733145511549/Put/seqid=0 2024-12-02T13:18:32,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742144_1320 (size=12301) 2024-12-02T13:18:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:32,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145512716, max timestamp=1733145512748Num uniques 859; 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145512716, max timestamp=1733145512748Num uniques 859; 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145512716, max timestamp=1733145512748Num uniques 859; 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:32,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:18:32,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145572778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145572780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145572782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,837 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c102adff57894cf181c13b2cc88ccba7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c102adff57894cf181c13b2cc88ccba7 2024-12-02T13:18:32,841 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into c102adff57894cf181c13b2cc88ccba7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:32,841 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:32,841 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145512386; duration=0sec 2024-12-02T13:18:32,842 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:32,842 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145572883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145572894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145572894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:32,903 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f60a8e843ce54bb29fffe54db518f025 2024-12-02T13:18:32,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/e0f4832f1fab4e7ea5fb718bb0f6963f is 50, key is test_row_0/B:col10/1733145511549/Put/seqid=0 2024-12-02T13:18:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742145_1321 (size=12301) 2024-12-02T13:18:32,912 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/e0f4832f1fab4e7ea5fb718bb0f6963f 2024-12-02T13:18:32,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c62860c8f997485c9a37abce078a3be4 is 50, key is test_row_0/C:col10/1733145511549/Put/seqid=0 2024-12-02T13:18:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742146_1322 (size=12301) 2024-12-02T13:18:33,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145573093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145573104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145573104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,322 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c62860c8f997485c9a37abce078a3be4 2024-12-02T13:18:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/f60a8e843ce54bb29fffe54db518f025 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025 2024-12-02T13:18:33,330 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T13:18:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/e0f4832f1fab4e7ea5fb718bb0f6963f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f 2024-12-02T13:18:33,334 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T13:18:33,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/c62860c8f997485c9a37abce078a3be4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4 2024-12-02T13:18:33,338 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T13:18:33,338 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 848ms, sequenceid=368, compaction requested=false 2024-12-02T13:18:33,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:33,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:33,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-12-02T13:18:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-12-02T13:18:33,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-02T13:18:33,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-12-02T13:18:33,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.6170 sec 2024-12-02T13:18:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:33,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:33,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/3e36f99ca477479292f8c965c072e87b is 50, key is test_row_0/A:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:33,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742147_1323 (size=14741) 2024-12-02T13:18:33,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/3e36f99ca477479292f8c965c072e87b 2024-12-02T13:18:33,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/bda5a6387fa54d72925131bc84442364 is 50, key is test_row_0/B:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:33,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742148_1324 (size=12301) 2024-12-02T13:18:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145573422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145573425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145573425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145573534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145573534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145573534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145573744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145573744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145573744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:33,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/bda5a6387fa54d72925131bc84442364 2024-12-02T13:18:33,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/5626f454d40d4699a9733d33277fd330 is 50, key is test_row_0/C:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742149_1325 (size=12301) 2024-12-02T13:18:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T13:18:33,874 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:33,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-12-02T13:18:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:33,876 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:33,877 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:33,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:34,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-12-02T13:18:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145574053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145574053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145574053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-12-02T13:18:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:34,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/5626f454d40d4699a9733d33277fd330 2024-12-02T13:18:34,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/3e36f99ca477479292f8c965c072e87b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b 2024-12-02T13:18:34,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b, entries=200, sequenceid=407, filesize=14.4 K 2024-12-02T13:18:34,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/bda5a6387fa54d72925131bc84442364 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364 2024-12-02T13:18:34,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364, entries=150, sequenceid=407, filesize=12.0 K 2024-12-02T13:18:34,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/5626f454d40d4699a9733d33277fd330 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330 2024-12-02T13:18:34,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330, entries=150, sequenceid=407, filesize=12.0 K 2024-12-02T13:18:34,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 847ms, sequenceid=407, compaction requested=true 2024-12-02T13:18:34,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:34,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:34,255 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:34,255 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,255 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,256 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15992ea7775244e0bf72c200ef89ba0c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=39.2 K 2024-12-02T13:18:34,256 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b35fcda9244d4b41842cd009c22841f3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.8 K 2024-12-02T13:18:34,256 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15992ea7775244e0bf72c200ef89ba0c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510384 2024-12-02T13:18:34,256 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b35fcda9244d4b41842cd009c22841f3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510384 2024-12-02T13:18:34,256 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f4832f1fab4e7ea5fb718bb0f6963f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733145511542 2024-12-02T13:18:34,256 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60a8e843ce54bb29fffe54db518f025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733145511542 2024-12-02T13:18:34,256 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bda5a6387fa54d72925131bc84442364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512776 2024-12-02T13:18:34,257 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e36f99ca477479292f8c965c072e87b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512774 2024-12-02T13:18:34,264 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:34,264 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:34,264 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/2157587cc14945aaa1f0117d276ea2b6 is 50, key is test_row_0/B:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:34,264 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/cdbd88c6438342a48b532d5aa112e7f7 is 50, key is test_row_0/A:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:34,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742151_1327 (size=13221) 2024-12-02T13:18:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742150_1326 (size=13221) 2024-12-02T13:18:34,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-12-02T13:18:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,334 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:34,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a3279c6b76094ba3b2e068ab8f0e1ec0 is 50, key is test_row_0/A:col10/1733145513424/Put/seqid=0 2024-12-02T13:18:34,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742152_1328 (size=12301) 2024-12-02T13:18:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:34,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:34,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145514582, max timestamp=1733145514588Num uniques 859; 2024-12-02T13:18:34,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:34,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145514582, max timestamp=1733145514588Num uniques 859; 2024-12-02T13:18:34,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:34,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145514582, max timestamp=1733145514588Num uniques 859; 2024-12-02T13:18:34,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:18:34,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145574611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145574615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145574616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,672 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/2157587cc14945aaa1f0117d276ea2b6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2157587cc14945aaa1f0117d276ea2b6 2024-12-02T13:18:34,673 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/cdbd88c6438342a48b532d5aa112e7f7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cdbd88c6438342a48b532d5aa112e7f7 2024-12-02T13:18:34,676 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into cdbd88c6438342a48b532d5aa112e7f7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:34,676 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into 2157587cc14945aaa1f0117d276ea2b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:34,676 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:34,676 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:34,677 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145514254; duration=0sec 2024-12-02T13:18:34,677 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145514255; duration=0sec 2024-12-02T13:18:34,677 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:34,677 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:34,677 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:34,677 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:34,677 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:34,678 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:34,678 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:34,678 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:34,678 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c102adff57894cf181c13b2cc88ccba7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.8 K 2024-12-02T13:18:34,678 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c102adff57894cf181c13b2cc88ccba7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733145510384 2024-12-02T13:18:34,678 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c62860c8f997485c9a37abce078a3be4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733145511542 2024-12-02T13:18:34,679 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5626f454d40d4699a9733d33277fd330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512776 2024-12-02T13:18:34,685 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:34,686 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/2c9597c1bc924ddcac877ed5242703eb is 50, key is test_row_0/C:col10/1733145512780/Put/seqid=0 2024-12-02T13:18:34,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742153_1329 (size=13221) 2024-12-02T13:18:34,702 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/2c9597c1bc924ddcac877ed5242703eb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2c9597c1bc924ddcac877ed5242703eb 2024-12-02T13:18:34,706 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 2c9597c1bc924ddcac877ed5242703eb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:34,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:34,706 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=13, startTime=1733145514255; duration=0sec 2024-12-02T13:18:34,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:34,707 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145574724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145574724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145574724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,741 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a3279c6b76094ba3b2e068ab8f0e1ec0 2024-12-02T13:18:34,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/45c5ca6b17db4c618c95c4b8bcc81a18 is 50, key is test_row_0/B:col10/1733145513424/Put/seqid=0 2024-12-02T13:18:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742154_1330 (size=12301) 2024-12-02T13:18:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145574933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145574933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:34,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145574934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:35,152 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/45c5ca6b17db4c618c95c4b8bcc81a18 2024-12-02T13:18:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/8ed6feb650ea43e8b27fafd878c3e30a is 50, key is test_row_0/C:col10/1733145513424/Put/seqid=0 2024-12-02T13:18:35,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742155_1331 (size=12301) 2024-12-02T13:18:35,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145575244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145575244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145575244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,567 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/8ed6feb650ea43e8b27fafd878c3e30a 2024-12-02T13:18:35,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/a3279c6b76094ba3b2e068ab8f0e1ec0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0 2024-12-02T13:18:35,581 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T13:18:35,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/45c5ca6b17db4c618c95c4b8bcc81a18 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18 2024-12-02T13:18:35,586 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T13:18:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/8ed6feb650ea43e8b27fafd878c3e30a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a 2024-12-02T13:18:35,591 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a, entries=150, sequenceid=417, filesize=12.0 K 2024-12-02T13:18:35,592 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1258ms, sequenceid=417, compaction requested=false 2024-12-02T13:18:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-12-02T13:18:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-12-02T13:18:35,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T13:18:35,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-12-02T13:18:35,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.7210 sec 2024-12-02T13:18:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:35,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:35,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:18:35,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d0d2e57a3e32489eba2f81ee43413c78 is 50, key is test_row_0/A:col10/1733145514614/Put/seqid=0 2024-12-02T13:18:35,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742156_1332 (size=14741) 2024-12-02T13:18:35,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145575763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145575764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145575767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145575873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145575873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145575873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-02T13:18:36,024 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:36,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-12-02T13:18:36,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:36,027 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:36,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145576084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145576084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145576084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:36,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d0d2e57a3e32489eba2f81ee43413c78 2024-12-02T13:18:36,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fd2a9a6f8b514efca499eb6d9143f184 is 50, key is test_row_0/B:col10/1733145514614/Put/seqid=0 2024-12-02T13:18:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742157_1333 (size=12301) 2024-12-02T13:18:36,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fd2a9a6f8b514efca499eb6d9143f184 2024-12-02T13:18:36,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0499785a4fc14c968fe4f6cd75e13098 is 50, key is test_row_0/C:col10/1733145514614/Put/seqid=0 2024-12-02T13:18:36,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742158_1334 (size=12301) 2024-12-02T13:18:36,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0499785a4fc14c968fe4f6cd75e13098 2024-12-02T13:18:36,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d0d2e57a3e32489eba2f81ee43413c78 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78 2024-12-02T13:18:36,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78, entries=200, sequenceid=458, filesize=14.4 K 2024-12-02T13:18:36,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/fd2a9a6f8b514efca499eb6d9143f184 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184 2024-12-02T13:18:36,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184, entries=150, sequenceid=458, filesize=12.0 K 2024-12-02T13:18:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0499785a4fc14c968fe4f6cd75e13098 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098 2024-12-02T13:18:36,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098, entries=150, sequenceid=458, filesize=12.0 K 2024-12-02T13:18:36,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 461ms, sequenceid=458, compaction requested=true 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:36,218 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:36,218 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:36,219 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:36,219 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:36,219 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:36,219 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:36,219 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,219 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cdbd88c6438342a48b532d5aa112e7f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=39.3 K 2024-12-02T13:18:36,219 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,219 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2157587cc14945aaa1f0117d276ea2b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=36.9 K 2024-12-02T13:18:36,220 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2157587cc14945aaa1f0117d276ea2b6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512776 2024-12-02T13:18:36,220 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdbd88c6438342a48b532d5aa112e7f7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512776 2024-12-02T13:18:36,220 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c5ca6b17db4c618c95c4b8bcc81a18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733145513418 2024-12-02T13:18:36,220 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3279c6b76094ba3b2e068ab8f0e1ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733145513418 2024-12-02T13:18:36,221 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2a9a6f8b514efca499eb6d9143f184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733145514610 2024-12-02T13:18:36,221 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d2e57a3e32489eba2f81ee43413c78, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733145514610 2024-12-02T13:18:36,229 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:36,229 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:36,229 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/290d7f228cfb479f87a451dd60bbae55 is 50, key is test_row_0/A:col10/1733145514614/Put/seqid=0 2024-12-02T13:18:36,230 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b95ac8ae1dad448899c3f45a51ae9cd1 is 50, key is test_row_0/B:col10/1733145514614/Put/seqid=0 2024-12-02T13:18:36,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742160_1336 (size=13323) 2024-12-02T13:18:36,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742159_1335 (size=13323) 2024-12-02T13:18:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:36,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:36,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5bb9c77071cf4187914096eb05484157 is 50, key is test_row_0/A:col10/1733145516274/Put/seqid=0 2024-12-02T13:18:36,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742161_1337 (size=14741) 2024-12-02T13:18:36,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5bb9c77071cf4187914096eb05484157 2024-12-02T13:18:36,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/af7c147528954f6587fbc90407cf0944 is 50, key is test_row_0/B:col10/1733145516274/Put/seqid=0 2024-12-02T13:18:36,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742162_1338 (size=12301) 2024-12-02T13:18:36,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/af7c147528954f6587fbc90407cf0944 2024-12-02T13:18:36,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6e45b07113e8458896967102dce6ac65 is 50, key is test_row_0/C:col10/1733145516274/Put/seqid=0 2024-12-02T13:18:36,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742163_1339 (size=12301) 2024-12-02T13:18:36,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6e45b07113e8458896967102dce6ac65 2024-12-02T13:18:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:36,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/5bb9c77071cf4187914096eb05484157 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157 2024-12-02T13:18:36,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157, entries=200, sequenceid=468, filesize=14.4 K 2024-12-02T13:18:36,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/af7c147528954f6587fbc90407cf0944 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944 2024-12-02T13:18:36,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944, entries=150, sequenceid=468, filesize=12.0 K 2024-12-02T13:18:36,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/6e45b07113e8458896967102dce6ac65 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65 2024-12-02T13:18:36,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65, entries=150, sequenceid=468, filesize=12.0 K 2024-12-02T13:18:36,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 84ms, sequenceid=468, compaction requested=true 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:36,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T13:18:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:36,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:18:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:36,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:36,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/e2a76476b5c44b9c8e1c736f45583dcb is 50, key is test_row_0/A:col10/1733145516365/Put/seqid=0 2024-12-02T13:18:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742164_1340 (size=14741) 2024-12-02T13:18:36,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/e2a76476b5c44b9c8e1c736f45583dcb 2024-12-02T13:18:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145576393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145576394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145576394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145576395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145576396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/520ec1733843499bb4e3e65f1ec50515 is 50, key is test_row_0/B:col10/1733145516365/Put/seqid=0 2024-12-02T13:18:36,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742165_1341 (size=12301) 2024-12-02T13:18:36,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145576503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145576503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,641 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/b95ac8ae1dad448899c3f45a51ae9cd1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b95ac8ae1dad448899c3f45a51ae9cd1 2024-12-02T13:18:36,641 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/290d7f228cfb479f87a451dd60bbae55 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/290d7f228cfb479f87a451dd60bbae55 2024-12-02T13:18:36,646 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into b95ac8ae1dad448899c3f45a51ae9cd1(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:36,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:36,646 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145516218; duration=0sec 2024-12-02T13:18:36,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T13:18:36,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:36,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. because compaction request was cancelled 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:36,647 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into 290d7f228cfb479f87a451dd60bbae55(size=13.0 K), total size for store is 27.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:36,647 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:36,648 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145516218; duration=0sec 2024-12-02T13:18:36,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:18:36,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:36,648 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:36,649 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:36,649 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:36,649 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-02T13:18:36,649 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:18:36,649 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. because compaction request was cancelled 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:36,650 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2c9597c1bc924ddcac877ed5242703eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=48.9 K 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9597c1bc924ddcac877ed5242703eb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733145512776 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed6feb650ea43e8b27fafd878c3e30a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733145513418 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. because compaction request was cancelled 2024-12-02T13:18:36,650 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:36,651 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0499785a4fc14c968fe4f6cd75e13098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733145514610 2024-12-02T13:18:36,651 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e45b07113e8458896967102dce6ac65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733145515765 2024-12-02T13:18:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:36,664 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:36,665 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0f5165f752af42efa5b1f6c3924d9bda is 50, key is test_row_0/C:col10/1733145516274/Put/seqid=0 2024-12-02T13:18:36,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742166_1342 (size=13357) 2024-12-02T13:18:36,677 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/0f5165f752af42efa5b1f6c3924d9bda as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0f5165f752af42efa5b1f6c3924d9bda 2024-12-02T13:18:36,681 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 0f5165f752af42efa5b1f6c3924d9bda(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:36,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:36,681 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=12, startTime=1733145516364; duration=0sec 2024-12-02T13:18:36,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:36,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145576714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145576714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:18:36,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/520ec1733843499bb4e3e65f1ec50515 2024-12-02T13:18:36,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/2f69d7554ef14dd4a4c6cd7036b84a0b is 50, key is test_row_0/C:col10/1733145516365/Put/seqid=0 2024-12-02T13:18:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742167_1343 (size=12301) 2024-12-02T13:18:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145576914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145576914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:36,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145576914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:36,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:36,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145577024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145577024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:37,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:37,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/2f69d7554ef14dd4a4c6cd7036b84a0b 2024-12-02T13:18:37,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/e2a76476b5c44b9c8e1c736f45583dcb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb 2024-12-02T13:18:37,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb, entries=200, sequenceid=487, filesize=14.4 K 2024-12-02T13:18:37,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/520ec1733843499bb4e3e65f1ec50515 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515 2024-12-02T13:18:37,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515, entries=150, sequenceid=487, filesize=12.0 K 2024-12-02T13:18:37,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/2f69d7554ef14dd4a4c6cd7036b84a0b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b 2024-12-02T13:18:37,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b, entries=150, sequenceid=487, filesize=12.0 K 2024-12-02T13:18:37,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 883ms, sequenceid=487, compaction requested=true 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:37,249 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:37,249 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:37,250 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:37,250 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:37,250 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:37,250 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:37,250 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:37,250 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:37,250 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b95ac8ae1dad448899c3f45a51ae9cd1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=37.0 K 2024-12-02T13:18:37,250 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/290d7f228cfb479f87a451dd60bbae55, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=41.8 K 2024-12-02T13:18:37,251 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b95ac8ae1dad448899c3f45a51ae9cd1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733145514610 2024-12-02T13:18:37,251 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 290d7f228cfb479f87a451dd60bbae55, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733145514610 2024-12-02T13:18:37,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-12-02T13:18:37,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:37,251 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:37,252 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting af7c147528954f6587fbc90407cf0944, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733145515765 2024-12-02T13:18:37,252 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb9c77071cf4187914096eb05484157, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733145515761 2024-12-02T13:18:37,253 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 520ec1733843499bb4e3e65f1ec50515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733145516351 2024-12-02T13:18:37,253 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a76476b5c44b9c8e1c736f45583dcb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733145516335 2024-12-02T13:18:37,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/eb71e349e23340bd8ce6114af9ef5392 is 50, key is test_row_0/A:col10/1733145516388/Put/seqid=0 2024-12-02T13:18:37,266 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:37,267 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/a44a123d66dd4195852e52c5a8bff590 is 50, key is test_row_0/B:col10/1733145516365/Put/seqid=0 2024-12-02T13:18:37,268 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:37,269 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/282fa1c31f9149b4a94899c55f6dd8a8 is 50, key is test_row_0/A:col10/1733145516365/Put/seqid=0 2024-12-02T13:18:37,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742168_1344 (size=12301) 2024-12-02T13:18:37,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742169_1345 (size=13425) 2024-12-02T13:18:37,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742170_1346 (size=13425) 2024-12-02T13:18:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:37,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. as already flushing 2024-12-02T13:18:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145577565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145577569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145577674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145577684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,687 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/eb71e349e23340bd8ce6114af9ef5392 2024-12-02T13:18:37,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f506299834a7438f82ee9acfc5f42da4 is 50, key is test_row_0/B:col10/1733145516388/Put/seqid=0 2024-12-02T13:18:37,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742171_1347 (size=12301) 2024-12-02T13:18:37,699 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/a44a123d66dd4195852e52c5a8bff590 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/a44a123d66dd4195852e52c5a8bff590 2024-12-02T13:18:37,703 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into a44a123d66dd4195852e52c5a8bff590(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:37,703 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:37,703 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145517249; duration=0sec 2024-12-02T13:18:37,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:37,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:37,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:18:37,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:18:37,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:18:37,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. because compaction request was cancelled 2024-12-02T13:18:37,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:37,707 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/282fa1c31f9149b4a94899c55f6dd8a8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/282fa1c31f9149b4a94899c55f6dd8a8 2024-12-02T13:18:37,711 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into 282fa1c31f9149b4a94899c55f6dd8a8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:37,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:37,711 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145517249; duration=0sec 2024-12-02T13:18:37,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:37,712 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:37,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145577884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145577893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34352 deadline: 1733145577924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34358 deadline: 1733145577924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,925 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:37,926 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34348 deadline: 1733145577934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:37,936 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in b75af96915a684ad1ebeb7dc1c1ba4d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:37,949 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-12-02T13:18:37,949 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:37,949 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,949 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:37,950 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-12-02T13:18:37,950 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:37,950 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,950 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:37,951 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-12-02T13:18:37,951 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:37,951 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,951 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,951 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:37,953 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-12-02T13:18:37,953 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:37,953 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-12-02T13:18:37,953 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,953 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:37,953 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,953 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,953 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:37,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:37,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:38,097 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f506299834a7438f82ee9acfc5f42da4 2024-12-02T13:18:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb3c231dfa70429da5595b1629ce0699 is 50, key is test_row_0/C:col10/1733145516388/Put/seqid=0 2024-12-02T13:18:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742172_1348 (size=12301) 2024-12-02T13:18:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34374 deadline: 1733145578194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:38,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34310 deadline: 1733145578204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:18:38,517 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb3c231dfa70429da5595b1629ce0699 2024-12-02T13:18:38,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/eb71e349e23340bd8ce6114af9ef5392 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392 2024-12-02T13:18:38,525 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392, entries=150, sequenceid=503, filesize=12.0 K 2024-12-02T13:18:38,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/f506299834a7438f82ee9acfc5f42da4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4 2024-12-02T13:18:38,531 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4, entries=150, sequenceid=503, filesize=12.0 K 2024-12-02T13:18:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb3c231dfa70429da5595b1629ce0699 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699 2024-12-02T13:18:38,536 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699, entries=150, sequenceid=503, filesize=12.0 K 2024-12-02T13:18:38,537 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1286ms, sequenceid=503, compaction requested=true 2024-12-02T13:18:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-12-02T13:18:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-12-02T13:18:38,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-02T13:18:38,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5110 sec 2024-12-02T13:18:38,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 2.5160 sec 2024-12-02T13:18:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:38,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:38,706 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:38,706 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:38,706 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:38,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:38,706 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:38,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:38,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/140116e0f95b4e4897f9f2045e5cb08a is 50, key is test_row_0/A:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:38,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742173_1349 (size=12301) 2024-12-02T13:18:38,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/140116e0f95b4e4897f9f2045e5cb08a 2024-12-02T13:18:38,715 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-12-02T13:18:38,715 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:38,715 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:38,716 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:38,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:38,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/07e0dc90236a490899649737e84dca83 is 50, key is test_row_0/B:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742174_1350 (size=12301) 2024-12-02T13:18:39,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/07e0dc90236a490899649737e84dca83 2024-12-02T13:18:39,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/59da3489a0834e00a08d55278165e021 is 50, key is test_row_0/C:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:39,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742175_1351 (size=12301) 2024-12-02T13:18:39,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/59da3489a0834e00a08d55278165e021 2024-12-02T13:18:39,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/140116e0f95b4e4897f9f2045e5cb08a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a 2024-12-02T13:18:39,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a, entries=150, sequenceid=525, filesize=12.0 K 2024-12-02T13:18:39,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/07e0dc90236a490899649737e84dca83 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83 2024-12-02T13:18:39,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83, entries=150, sequenceid=525, filesize=12.0 K 2024-12-02T13:18:39,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/59da3489a0834e00a08d55278165e021 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021 2024-12-02T13:18:39,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021, entries=150, sequenceid=525, filesize=12.0 K 2024-12-02T13:18:39,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 862ms, sequenceid=525, compaction requested=true 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:39,569 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:39,569 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b75af96915a684ad1ebeb7dc1c1ba4d3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:39,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:39,570 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:39,570 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:39,570 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/B is initiating minor compaction (all files) 2024-12-02T13:18:39,570 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/A is initiating minor compaction (all files) 2024-12-02T13:18:39,570 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/B in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:39,570 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/A in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:39,570 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/a44a123d66dd4195852e52c5a8bff590, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=37.1 K 2024-12-02T13:18:39,570 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/282fa1c31f9149b4a94899c55f6dd8a8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=37.1 K 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282fa1c31f9149b4a94899c55f6dd8a8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733145516351 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a44a123d66dd4195852e52c5a8bff590, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733145516351 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f506299834a7438f82ee9acfc5f42da4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1733145516388 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb71e349e23340bd8ce6114af9ef5392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1733145516388 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e0dc90236a490899649737e84dca83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733145517559 2024-12-02T13:18:39,571 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140116e0f95b4e4897f9f2045e5cb08a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733145517559 2024-12-02T13:18:39,579 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#B#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:39,579 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:39,579 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/2ca68bcbadf540f181f800cdc753e026 is 50, key is test_row_0/B:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:39,579 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d4f8b72909dd4588919e22729eb60d1e is 50, key is test_row_0/A:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742177_1353 (size=13527) 2024-12-02T13:18:39,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742176_1352 (size=13527) 2024-12-02T13:18:39,588 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/d4f8b72909dd4588919e22729eb60d1e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d4f8b72909dd4588919e22729eb60d1e 2024-12-02T13:18:39,592 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/A of b75af96915a684ad1ebeb7dc1c1ba4d3 into d4f8b72909dd4588919e22729eb60d1e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:39,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:39,592 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/A, priority=13, startTime=1733145519569; duration=0sec 2024-12-02T13:18:39,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:39,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:A 2024-12-02T13:18:39,592 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:18:39,594 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:18:39,594 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): b75af96915a684ad1ebeb7dc1c1ba4d3/C is initiating minor compaction (all files) 2024-12-02T13:18:39,594 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b75af96915a684ad1ebeb7dc1c1ba4d3/C in TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:39,594 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0f5165f752af42efa5b1f6c3924d9bda, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp, totalSize=49.1 K 2024-12-02T13:18:39,594 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f5165f752af42efa5b1f6c3924d9bda, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733145515765 2024-12-02T13:18:39,595 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f69d7554ef14dd4a4c6cd7036b84a0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1733145516351 2024-12-02T13:18:39,595 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb3c231dfa70429da5595b1629ce0699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1733145516388 2024-12-02T13:18:39,595 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59da3489a0834e00a08d55278165e021, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733145517559 2024-12-02T13:18:39,603 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b75af96915a684ad1ebeb7dc1c1ba4d3#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:39,604 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/4e651963607e4835b8730b4a1128e2ee is 50, key is test_row_0/C:col10/1733145517567/Put/seqid=0 2024-12-02T13:18:39,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742178_1354 (size=13493) 2024-12-02T13:18:39,938 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-12-02T13:18:39,938 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:39,939 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,939 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:39,959 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-12-02T13:18:39,959 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-12-02T13:18:39,959 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:39,959 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:18:39,959 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,959 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,959 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,959 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:39,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:39,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:39,993 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/2ca68bcbadf540f181f800cdc753e026 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2ca68bcbadf540f181f800cdc753e026 2024-12-02T13:18:39,999 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/B of b75af96915a684ad1ebeb7dc1c1ba4d3 into 2ca68bcbadf540f181f800cdc753e026(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:39,999 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:39,999 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/B, priority=13, startTime=1733145519569; duration=0sec 2024-12-02T13:18:39,999 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:39,999 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:B 2024-12-02T13:18:40,015 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/4e651963607e4835b8730b4a1128e2ee as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4e651963607e4835b8730b4a1128e2ee 2024-12-02T13:18:40,022 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b75af96915a684ad1ebeb7dc1c1ba4d3/C of b75af96915a684ad1ebeb7dc1c1ba4d3 into 4e651963607e4835b8730b4a1128e2ee(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:40,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b75af96915a684ad1ebeb7dc1c1ba4d3: 2024-12-02T13:18:40,022 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., storeName=b75af96915a684ad1ebeb7dc1c1ba4d3/C, priority=12, startTime=1733145519569; duration=0sec 2024-12-02T13:18:40,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:40,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b75af96915a684ad1ebeb7dc1c1ba4d3:C 2024-12-02T13:18:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T13:18:40,195 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3694 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3780 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3639 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3665 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3765 2024-12-02T13:18:40,195 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:18:40,195 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:18:40,196 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:18:40,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:40,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:40,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:18:40,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:18:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:40,203 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145520203"}]},"ts":"1733145520203"} 2024-12-02T13:18:40,205 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:18:40,205 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:18:40,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:18:40,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, UNASSIGN}] 2024-12-02T13:18:40,209 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, UNASSIGN 2024-12-02T13:18:40,210 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b75af96915a684ad1ebeb7dc1c1ba4d3, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:40,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, UNASSIGN because future has completed 2024-12-02T13:18:40,212 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:18:40,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:40,365 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:40,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:18:40,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing b75af96915a684ad1ebeb7dc1c1ba4d3, disabling compactions & flushes 2024-12-02T13:18:40,365 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:40,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:40,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. after waiting 0 ms 2024-12-02T13:18:40,365 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:40,365 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing b75af96915a684ad1ebeb7dc1c1ba4d3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=A 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=B 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b75af96915a684ad1ebeb7dc1c1ba4d3, store=C 2024-12-02T13:18:40,366 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:40,372 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2b6a4ca888ae47b29acfc2d8be6eaf89 is 50, key is test_row_0/A:col10/1733145519956/Put/seqid=0 2024-12-02T13:18:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742179_1355 (size=12301) 2024-12-02T13:18:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:40,778 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2b6a4ca888ae47b29acfc2d8be6eaf89 2024-12-02T13:18:40,786 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d346b8862ca54ce6adad25ef7d518cef is 50, key is test_row_0/B:col10/1733145519956/Put/seqid=0 2024-12-02T13:18:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742180_1356 (size=12301) 2024-12-02T13:18:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:41,190 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d346b8862ca54ce6adad25ef7d518cef 2024-12-02T13:18:41,203 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb85b56771764941960ba12e74d4ad30 is 50, key is test_row_0/C:col10/1733145519956/Put/seqid=0 2024-12-02T13:18:41,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742181_1357 (size=12301) 2024-12-02T13:18:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:41,609 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb85b56771764941960ba12e74d4ad30 2024-12-02T13:18:41,621 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/A/2b6a4ca888ae47b29acfc2d8be6eaf89 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2b6a4ca888ae47b29acfc2d8be6eaf89 2024-12-02T13:18:41,626 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2b6a4ca888ae47b29acfc2d8be6eaf89, entries=150, sequenceid=535, filesize=12.0 K 2024-12-02T13:18:41,627 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/B/d346b8862ca54ce6adad25ef7d518cef as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d346b8862ca54ce6adad25ef7d518cef 2024-12-02T13:18:41,631 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d346b8862ca54ce6adad25ef7d518cef, entries=150, sequenceid=535, filesize=12.0 K 2024-12-02T13:18:41,632 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/.tmp/C/eb85b56771764941960ba12e74d4ad30 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb85b56771764941960ba12e74d4ad30 2024-12-02T13:18:41,635 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb85b56771764941960ba12e74d4ad30, entries=150, sequenceid=535, filesize=12.0 K 2024-12-02T13:18:41,636 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1271ms, sequenceid=535, compaction requested=false 2024-12-02T13:18:41,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f37e10de16bf49b99a4d519b09959e0f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cbd5a3a85c0e4c1493e35a301e4e6692, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/df8dbe11cdd64b0c8b57b859497fe5c1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9aa6dc4c734448f1bf20b4619f904440, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a4478757bf4d470381b49edf22876cb4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0bd3479fe7ac4e9ca717b256c2645891, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15992ea7775244e0bf72c200ef89ba0c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cdbd88c6438342a48b532d5aa112e7f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/290d7f228cfb479f87a451dd60bbae55, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/282fa1c31f9149b4a94899c55f6dd8a8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a] to archive 2024-12-02T13:18:41,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:41,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/4492c53b7aac43c098390bebe7752d18 2024-12-02T13:18:41,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a66f15b3f5c0488880aadcad35af349e 2024-12-02T13:18:41,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f37e10de16bf49b99a4d519b09959e0f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f37e10de16bf49b99a4d519b09959e0f 2024-12-02T13:18:41,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2ed272c9a14c492589f8569b06095f3a 2024-12-02T13:18:41,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15dfbf9016b2441f96f9635a0e81d13b 2024-12-02T13:18:41,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ff8bf2027e3f45b1a34fcbccf7b61961 2024-12-02T13:18:41,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cbd5a3a85c0e4c1493e35a301e4e6692 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cbd5a3a85c0e4c1493e35a301e4e6692 2024-12-02T13:18:41,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a0064a1e1cef4336bb0beda8817e2834 2024-12-02T13:18:41,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/df8dbe11cdd64b0c8b57b859497fe5c1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/df8dbe11cdd64b0c8b57b859497fe5c1 2024-12-02T13:18:41,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9798c999c59243c29f78a5511567c732 2024-12-02T13:18:41,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/31e7bd1a749c4a3cbd14e5a27e39f62e 2024-12-02T13:18:41,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f077398ba3614315bf12c441ba270663 2024-12-02T13:18:41,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b0f67ccde1ee4cf0917815fa1ca4d7c0 2024-12-02T13:18:41,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5b16a59de92b4bb980407ea547d437d3 2024-12-02T13:18:41,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9aa6dc4c734448f1bf20b4619f904440 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9aa6dc4c734448f1bf20b4619f904440 2024-12-02T13:18:41,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0e6d1c0a80834cacad320697a6582599 2024-12-02T13:18:41,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/9c24543b6fa64a04928d1bfdb423e09b 2024-12-02T13:18:41,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a4478757bf4d470381b49edf22876cb4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a4478757bf4d470381b49edf22876cb4 2024-12-02T13:18:41,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/ed1c0d07cb5d4ed2846cd12ca0e0c29e 2024-12-02T13:18:41,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d19ab4b32b5b4a1d8334b7d89b7d671b 2024-12-02T13:18:41,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d3a7a844d4964516bf47cf82642f97fa 2024-12-02T13:18:41,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0bd3479fe7ac4e9ca717b256c2645891 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/0bd3479fe7ac4e9ca717b256c2645891 2024-12-02T13:18:41,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f5ae102175d84d2fbff37f324a6f5b20 2024-12-02T13:18:41,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/b59acd367ea24973b54de8f1495686ee 2024-12-02T13:18:41,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15992ea7775244e0bf72c200ef89ba0c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/15992ea7775244e0bf72c200ef89ba0c 2024-12-02T13:18:41,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/f60a8e843ce54bb29fffe54db518f025 2024-12-02T13:18:41,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/3e36f99ca477479292f8c965c072e87b 2024-12-02T13:18:41,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cdbd88c6438342a48b532d5aa112e7f7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/cdbd88c6438342a48b532d5aa112e7f7 2024-12-02T13:18:41,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/a3279c6b76094ba3b2e068ab8f0e1ec0 2024-12-02T13:18:41,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d0d2e57a3e32489eba2f81ee43413c78 2024-12-02T13:18:41,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/290d7f228cfb479f87a451dd60bbae55 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/290d7f228cfb479f87a451dd60bbae55 2024-12-02T13:18:41,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/5bb9c77071cf4187914096eb05484157 2024-12-02T13:18:41,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/e2a76476b5c44b9c8e1c736f45583dcb 2024-12-02T13:18:41,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/282fa1c31f9149b4a94899c55f6dd8a8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/282fa1c31f9149b4a94899c55f6dd8a8 2024-12-02T13:18:41,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/eb71e349e23340bd8ce6114af9ef5392 2024-12-02T13:18:41,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/140116e0f95b4e4897f9f2045e5cb08a 2024-12-02T13:18:41,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b5468e2d63694e67bab9a3fbc3d2b97a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/02bb90be0d3347fc82e749c6fe942e94, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/790a53bb99c446f4a3953c74b987679b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fceeb73c399f46808f7babf5ac748a32, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/74bd7cab83104112910da1e100c1320a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b35fcda9244d4b41842cd009c22841f3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2157587cc14945aaa1f0117d276ea2b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b95ac8ae1dad448899c3f45a51ae9cd1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/a44a123d66dd4195852e52c5a8bff590, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83] to archive 2024-12-02T13:18:41,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:41,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/78fbb9b1519a468f9ba9188c08b7c34e 2024-12-02T13:18:41,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/3afadfc8ac1e43f79438357cdf0750c3 2024-12-02T13:18:41,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/c49f0a6c79ac4c8485b2c3c3fe4d4a2f 2024-12-02T13:18:41,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b4007f06e7c54136b2dce4900c9d7e0f 2024-12-02T13:18:41,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/05510bbf66f94b6a94a615f72c6251d2 2024-12-02T13:18:41,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b5468e2d63694e67bab9a3fbc3d2b97a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b5468e2d63694e67bab9a3fbc3d2b97a 2024-12-02T13:18:41,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/611b1f7566d84cf5be9ae66f01c9ac3c 2024-12-02T13:18:41,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ee9fd0ba8bb84a02ad440dddbcb5b775 2024-12-02T13:18:41,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/02bb90be0d3347fc82e749c6fe942e94 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/02bb90be0d3347fc82e749c6fe942e94 2024-12-02T13:18:41,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d6fecc203880451582c4c5968ba947d7 2024-12-02T13:18:41,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/456cad7b634c4e9fa329e6855ae8b056 2024-12-02T13:18:41,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/774bc102ba0e40c4a10925f79573c974 2024-12-02T13:18:41,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/0ae381403edd4695bcca8210f5fa7e62 2024-12-02T13:18:41,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/790a53bb99c446f4a3953c74b987679b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/790a53bb99c446f4a3953c74b987679b 2024-12-02T13:18:41,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/228ee67c25644b5aba32edee25767391 2024-12-02T13:18:41,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fa547d98a38a411c9e92216a89989251 2024-12-02T13:18:41,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fceeb73c399f46808f7babf5ac748a32 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fceeb73c399f46808f7babf5ac748a32 2024-12-02T13:18:41,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/7a5f59ed2f084b8db7cb99e6a6e6581e 2024-12-02T13:18:41,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/61166c296d3a42a99768e80648dcbb9b 2024-12-02T13:18:41,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f9a02b3211c946c597df99e78a282be4 2024-12-02T13:18:41,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/74bd7cab83104112910da1e100c1320a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/74bd7cab83104112910da1e100c1320a 2024-12-02T13:18:41,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b3fa64ba3f0f474d8663f2d6f1c7c9d1 2024-12-02T13:18:41,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/ec42c864c90f44b69f321d08af7df1ec 2024-12-02T13:18:41,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b35fcda9244d4b41842cd009c22841f3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b35fcda9244d4b41842cd009c22841f3 2024-12-02T13:18:41,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/1bf04cce630a4872808d591f6661caa1 2024-12-02T13:18:41,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/e0f4832f1fab4e7ea5fb718bb0f6963f 2024-12-02T13:18:41,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2157587cc14945aaa1f0117d276ea2b6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2157587cc14945aaa1f0117d276ea2b6 2024-12-02T13:18:41,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/bda5a6387fa54d72925131bc84442364 2024-12-02T13:18:41,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/45c5ca6b17db4c618c95c4b8bcc81a18 2024-12-02T13:18:41,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b95ac8ae1dad448899c3f45a51ae9cd1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/b95ac8ae1dad448899c3f45a51ae9cd1 2024-12-02T13:18:41,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/fd2a9a6f8b514efca499eb6d9143f184 2024-12-02T13:18:41,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/af7c147528954f6587fbc90407cf0944 2024-12-02T13:18:41,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/a44a123d66dd4195852e52c5a8bff590 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/a44a123d66dd4195852e52c5a8bff590 2024-12-02T13:18:41,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/520ec1733843499bb4e3e65f1ec50515 2024-12-02T13:18:41,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/f506299834a7438f82ee9acfc5f42da4 2024-12-02T13:18:41,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/07e0dc90236a490899649737e84dca83 2024-12-02T13:18:41,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0d273c74ed134bf4bf2415e77c9d59e3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/51d5080c8043407cbace620b6fb728a5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/e567ae9397234feaa67032332ceae94a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/97aad5750b314d00b5155e19e850cd0e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/092c1596772742e4b61866bcda1f21a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/fe710033d37841288100ec51fd332446, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c102adff57894cf181c13b2cc88ccba7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2c9597c1bc924ddcac877ed5242703eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0f5165f752af42efa5b1f6c3924d9bda, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021] to archive 2024-12-02T13:18:41,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:18:41,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4d1e2708b4bc49f3af6951f18710e833 2024-12-02T13:18:41,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c0762efa64fc4549aeb643d2c479ceeb 2024-12-02T13:18:41,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0d273c74ed134bf4bf2415e77c9d59e3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0d273c74ed134bf4bf2415e77c9d59e3 2024-12-02T13:18:41,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/90b174ecdd564c33a63af75f451b4017 2024-12-02T13:18:41,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6ef0cc897c3a4e0884251e5e11c730d2 2024-12-02T13:18:41,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/51d5080c8043407cbace620b6fb728a5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/51d5080c8043407cbace620b6fb728a5 2024-12-02T13:18:41,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/36b38fa2a8ff4fdcb4b8f947de45d7f3 2024-12-02T13:18:41,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/355fceb69b6243e0822518046e3a470e 2024-12-02T13:18:41,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/e567ae9397234feaa67032332ceae94a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/e567ae9397234feaa67032332ceae94a 2024-12-02T13:18:41,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9f989250de343e7b425c619a7d71cfd 2024-12-02T13:18:41,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/f9d8395ec0044dd38529de4dc1226d3e 2024-12-02T13:18:41,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/86efbde0036f4d189522d5b2f4e4c551 2024-12-02T13:18:41,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/7922cb80885040159e89beb04205a0dd 2024-12-02T13:18:41,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/97aad5750b314d00b5155e19e850cd0e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/97aad5750b314d00b5155e19e850cd0e 2024-12-02T13:18:41,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/535177b893f948de9f9b9780c7622219 2024-12-02T13:18:41,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/24e2f4ee717a4ad2bc67c19d0e7d7b50 2024-12-02T13:18:41,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/092c1596772742e4b61866bcda1f21a6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/092c1596772742e4b61866bcda1f21a6 2024-12-02T13:18:41,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/df4ad470d3ea4b3eb80e406bd66d5dee 2024-12-02T13:18:41,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/3f0c2551a728497eacc7509e6da2d165 2024-12-02T13:18:41,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/ed13eed6a53b41f1b76abe4c546cb976 2024-12-02T13:18:41,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/fe710033d37841288100ec51fd332446 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/fe710033d37841288100ec51fd332446 2024-12-02T13:18:41,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c47a7ee66d4c4f178d5fc00b63e24b8c 2024-12-02T13:18:41,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/66dc5858f5234af782dd836c62de0957 2024-12-02T13:18:41,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c102adff57894cf181c13b2cc88ccba7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c102adff57894cf181c13b2cc88ccba7 2024-12-02T13:18:41,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/77c89d2c4a8d4bea8dd1fbd0a26d1cd0 2024-12-02T13:18:41,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/c62860c8f997485c9a37abce078a3be4 2024-12-02T13:18:41,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2c9597c1bc924ddcac877ed5242703eb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2c9597c1bc924ddcac877ed5242703eb 2024-12-02T13:18:41,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/5626f454d40d4699a9733d33277fd330 2024-12-02T13:18:41,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/8ed6feb650ea43e8b27fafd878c3e30a 2024-12-02T13:18:41,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0499785a4fc14c968fe4f6cd75e13098 2024-12-02T13:18:41,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0f5165f752af42efa5b1f6c3924d9bda to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/0f5165f752af42efa5b1f6c3924d9bda 2024-12-02T13:18:41,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/6e45b07113e8458896967102dce6ac65 2024-12-02T13:18:41,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/2f69d7554ef14dd4a4c6cd7036b84a0b 2024-12-02T13:18:41,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb3c231dfa70429da5595b1629ce0699 2024-12-02T13:18:41,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/59da3489a0834e00a08d55278165e021 2024-12-02T13:18:41,724 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/recovered.edits/538.seqid, newMaxSeqId=538, maxSeqId=1 2024-12-02T13:18:41,725 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. 2024-12-02T13:18:41,725 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for b75af96915a684ad1ebeb7dc1c1ba4d3: Waiting for close lock at 1733145520365Running coprocessor pre-close hooks at 1733145520365Disabling compacts and flushes for region at 1733145520365Disabling writes for close at 1733145520365Obtaining lock to block concurrent updates at 1733145520365Preparing flush snapshotting stores in b75af96915a684ad1ebeb7dc1c1ba4d3 at 1733145520365Finished memstore snapshotting TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733145520366 (+1 ms)Flushing stores of TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3. at 1733145520367 (+1 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/A: creating writer at 1733145520367Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/A: appending metadata at 1733145520371 (+4 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/A: closing flushed file at 1733145520371Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/B: creating writer at 1733145520782 (+411 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/B: appending metadata at 1733145520785 (+3 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/B: closing flushed file at 1733145520785Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/C: creating writer at 1733145521198 (+413 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/C: appending metadata at 1733145521202 (+4 ms)Flushing b75af96915a684ad1ebeb7dc1c1ba4d3/C: closing flushed file at 1733145521202Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a63873: reopening flushed file at 1733145521619 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50fb81af: reopening flushed file at 1733145521626 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bfc101e: reopening flushed file at 1733145521631 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b75af96915a684ad1ebeb7dc1c1ba4d3 in 1271ms, sequenceid=535, compaction requested=false at 1733145521636 (+5 ms)Writing region close event to WAL at 1733145521722 (+86 ms)Running coprocessor post-close hooks at 1733145521725 (+3 ms)Closed at 1733145521725 2024-12-02T13:18:41,726 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:41,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b75af96915a684ad1ebeb7dc1c1ba4d3, regionState=CLOSED 2024-12-02T13:18:41,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:41,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-02T13:18:41,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure b75af96915a684ad1ebeb7dc1c1ba4d3, server=06e3e18f2515,45693,1733145428280 in 1.5160 sec 2024-12-02T13:18:41,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-02T13:18:41,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b75af96915a684ad1ebeb7dc1c1ba4d3, UNASSIGN in 1.5220 sec 2024-12-02T13:18:41,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-02T13:18:41,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5260 sec 2024-12-02T13:18:41,734 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145521734"}]},"ts":"1733145521734"} 2024-12-02T13:18:41,735 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:18:41,735 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:18:41,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5370 sec 2024-12-02T13:18:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T13:18:42,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:42,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:18:42,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,362 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-12-02T13:18:42,363 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,366 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:42,368 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/recovered.edits] 2024-12-02T13:18:42,371 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2b6a4ca888ae47b29acfc2d8be6eaf89 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/2b6a4ca888ae47b29acfc2d8be6eaf89 2024-12-02T13:18:42,372 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d4f8b72909dd4588919e22729eb60d1e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/A/d4f8b72909dd4588919e22729eb60d1e 2024-12-02T13:18:42,375 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2ca68bcbadf540f181f800cdc753e026 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/2ca68bcbadf540f181f800cdc753e026 2024-12-02T13:18:42,376 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d346b8862ca54ce6adad25ef7d518cef to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/B/d346b8862ca54ce6adad25ef7d518cef 2024-12-02T13:18:42,378 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4e651963607e4835b8730b4a1128e2ee to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/4e651963607e4835b8730b4a1128e2ee 2024-12-02T13:18:42,379 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb85b56771764941960ba12e74d4ad30 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/C/eb85b56771764941960ba12e74d4ad30 2024-12-02T13:18:42,382 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/recovered.edits/538.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3/recovered.edits/538.seqid 2024-12-02T13:18:42,383 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/b75af96915a684ad1ebeb7dc1c1ba4d3 2024-12-02T13:18:42,383 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:18:42,386 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,389 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:18:42,391 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:18:42,392 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,392 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:18:42,393 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145522392"}]},"ts":"9223372036854775807"} 2024-12-02T13:18:42,395 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:18:42,395 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b75af96915a684ad1ebeb7dc1c1ba4d3, NAME => 'TestAcidGuarantees,,1733145495730.b75af96915a684ad1ebeb7dc1c1ba4d3.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:18:42,395 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:18:42,395 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145522395"}]},"ts":"9223372036854775807"} 2024-12-02T13:18:42,397 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:18:42,398 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-12-02T13:18:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-12-02T13:18:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:18:42,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:42,487 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_740994727_22 at /127.0.0.1:51730 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-130320354_22 at /127.0.0.1:56318 [Waiting for operation #395] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_740994727_22 at /127.0.0.1:40940 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-130320354_22 at /127.0.0.1:40938 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=499 (was 512), ProcessCount=11 (was 11), AvailableMemoryMB=2580 (was 1490) - AvailableMemoryMB LEAK? - 2024-12-02T13:18:42,494 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=499, ProcessCount=11, AvailableMemoryMB=2579 2024-12-02T13:18:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:18:42,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:18:42,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:42,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:18:42,502 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:42,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-12-02T13:18:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:42,502 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:18:42,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742182_1358 (size=1061) 2024-12-02T13:18:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:42,913 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:18:42,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742183_1359 (size=53) 2024-12-02T13:18:43,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2fbac5b7f28671050128676c4e02d4f1, disabling compactions & flushes 2024-12-02T13:18:43,326 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. after waiting 0 ms 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,326 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2fbac5b7f28671050128676c4e02d4f1: Waiting for close lock at 1733145523326Disabling compacts and flushes for region at 1733145523326Disabling writes for close at 1733145523326Writing region close event to WAL at 1733145523326Closed at 1733145523326 2024-12-02T13:18:43,329 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:18:43,330 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145523329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145523329"}]},"ts":"1733145523329"} 2024-12-02T13:18:43,333 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:18:43,335 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:18:43,335 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145523335"}]},"ts":"1733145523335"} 2024-12-02T13:18:43,337 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:18:43,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, ASSIGN}] 2024-12-02T13:18:43,339 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, ASSIGN 2024-12-02T13:18:43,339 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:18:43,490 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:43,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, ASSIGN because future has completed 2024-12-02T13:18:43,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:43,655 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,655 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:18:43,656 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,656 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:18:43,657 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,657 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,660 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,661 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:43,662 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName A 2024-12-02T13:18:43,662 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:43,662 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:43,663 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,664 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:43,664 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName B 2024-12-02T13:18:43,664 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:43,665 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:43,665 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,665 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:43,666 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName C 2024-12-02T13:18:43,666 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:43,666 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:43,666 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,666 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,667 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,667 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,668 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,668 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,668 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:18:43,669 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,671 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:18:43,671 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 2fbac5b7f28671050128676c4e02d4f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73930458, jitterRate=0.10164967179298401}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:18:43,671 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:43,671 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 2fbac5b7f28671050128676c4e02d4f1: Running coprocessor pre-open hook at 1733145523657Writing region info on filesystem at 1733145523657Initializing all the Stores at 1733145523659 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145523659Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145523660 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145523660Cleaning up temporary data from old regions at 1733145523668 (+8 ms)Running coprocessor post-open hooks at 1733145523671 (+3 ms)Region opened successfully at 1733145523671 2024-12-02T13:18:43,672 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., pid=99, masterSystemTime=1733145523648 2024-12-02T13:18:43,673 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,674 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:43,674 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:43,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:43,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-12-02T13:18:43,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 in 182 msec 2024-12-02T13:18:43,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-02T13:18:43,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, ASSIGN in 340 msec 2024-12-02T13:18:43,679 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:18:43,679 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145523679"}]},"ts":"1733145523679"} 2024-12-02T13:18:43,680 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:18:43,680 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:18:43,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-12-02T13:18:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T13:18:44,656 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:44,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:44,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:44,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:44,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dd6e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:44,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6729b197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:44,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:44,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:44,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:18:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:18:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11f441ca 2024-12-02T13:18:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:18:44,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:18:44,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:18:44,672 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:18:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T13:18:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742184_1360 (size=1097) 2024-12-02T13:18:45,088 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T13:18:45,089 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T13:18:45,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:18:45,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE}] 2024-12-02T13:18:45,098 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE 2024-12-02T13:18:45,099 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE because future has completed 2024-12-02T13:18:45,102 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:18:45,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:45,256 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,256 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:18:45,256 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 2fbac5b7f28671050128676c4e02d4f1, disabling compactions & flushes 2024-12-02T13:18:45,257 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,257 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,257 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. after waiting 0 ms 2024-12-02T13:18:45,257 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,266 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T13:18:45,267 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,267 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 2fbac5b7f28671050128676c4e02d4f1: Waiting for close lock at 1733145525256Running coprocessor pre-close hooks at 1733145525256Disabling compacts and flushes for region at 1733145525256Disabling writes for close at 1733145525257 (+1 ms)Writing region close event to WAL at 1733145525258 (+1 ms)Running coprocessor post-close hooks at 1733145525266 (+8 ms)Closed at 1733145525267 (+1 ms) 2024-12-02T13:18:45,267 WARN [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 2fbac5b7f28671050128676c4e02d4f1 to self. 2024-12-02T13:18:45,269 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=CLOSED 2024-12-02T13:18:45,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:45,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-12-02T13:18:45,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 in 169 msec 2024-12-02T13:18:45,272 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE; state=CLOSED, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=true 2024-12-02T13:18:45,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE because future has completed 2024-12-02T13:18:45,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:18:45,585 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,585 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:18:45,586 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,586 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:18:45,586 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,586 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,589 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,591 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:45,592 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName A 2024-12-02T13:18:45,593 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:45,593 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:45,594 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,594 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:45,594 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName B 2024-12-02T13:18:45,594 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:45,595 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:45,595 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,595 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:18:45,595 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fbac5b7f28671050128676c4e02d4f1 columnFamilyName C 2024-12-02T13:18:45,595 DEBUG [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:45,596 INFO [StoreOpener-2fbac5b7f28671050128676c4e02d4f1-1 {}] regionserver.HStore(327): Store=2fbac5b7f28671050128676c4e02d4f1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:18:45,596 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,596 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,596 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,597 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,598 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,598 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,598 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:18:45,600 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,600 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 2fbac5b7f28671050128676c4e02d4f1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72560705, jitterRate=0.0812387615442276}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:18:45,600 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,601 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 2fbac5b7f28671050128676c4e02d4f1: Running coprocessor pre-open hook at 1733145525586Writing region info on filesystem at 1733145525586Initializing all the Stores at 1733145525588 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145525588Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145525589 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145525589Cleaning up temporary data from old regions at 1733145525598 (+9 ms)Running coprocessor post-open hooks at 1733145525600 (+2 ms)Region opened successfully at 1733145525601 (+1 ms) 2024-12-02T13:18:45,601 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., pid=104, masterSystemTime=1733145525578 2024-12-02T13:18:45,603 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,603 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,603 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=OPEN, openSeqNum=5, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:18:45,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-12-02T13:18:45,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 in 180 msec 2024-12-02T13:18:45,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T13:18:45,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, REOPEN/MOVE in 510 msec 2024-12-02T13:18:45,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-12-02T13:18:45,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-12-02T13:18:45,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-12-02T13:18:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-02T13:18:45,614 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:45,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,616 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da1c474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,617 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c61716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,619 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac89a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b13dbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e60aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c19069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b513b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,627 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba49c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,630 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1183e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30818879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5bb59e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29138a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,639 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722de190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43af66da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,643 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4787234a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,646 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682bc8ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5bf869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,649 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65164f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:18:45,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:18:45,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:18:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:18:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:18:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab7525f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:18:45,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:18:45,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:18:45,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:18:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c422e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:18:45,654 DEBUG [Thread-1642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,654 DEBUG [Thread-1644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,654 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,655 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,655 DEBUG [Thread-1640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,656 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,656 DEBUG [Thread-1640 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:45,656 DEBUG [Thread-1648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,657 DEBUG [Thread-1640 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,658 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,658 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T13:18:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,661 DEBUG [Thread-1646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:18:45,661 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:18:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:18:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,663 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:18:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,664 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:45,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:18:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:45,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202abc70673b179406cb1e03a192b80832e_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145525671/Put/seqid=0 2024-12-02T13:18:45,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742185_1361 (size=12154) 2024-12-02T13:18:45,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:45,717 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202abc70673b179406cb1e03a192b80832e_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202abc70673b179406cb1e03a192b80832e_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:45,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145585710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145525684, max timestamp=1733145525715Num uniques 859; 2024-12-02T13:18:45,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145525684, max timestamp=1733145525715Num uniques 859; 2024-12-02T13:18:45,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:18:45,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:18:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145525684, max timestamp=1733145525715Num uniques 859; 2024-12-02T13:18:45,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:18:45,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/d554de96a5dd4a0d90cda44aeecac84c, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:45,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/d554de96a5dd4a0d90cda44aeecac84c is 175, key is test_row_0/A:col10/1733145525671/Put/seqid=0 2024-12-02T13:18:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742186_1362 (size=30955) 2024-12-02T13:18:45,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145585743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145585745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145585746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145585747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:45,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:45,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:45,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145585824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145585853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145585864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145585864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145585864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:45,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:45,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:45,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:46,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145586034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145586064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145586074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145586075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145586075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:46,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,130 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/d554de96a5dd4a0d90cda44aeecac84c 2024-12-02T13:18:46,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2b096612b27e4f29a6046faba444e63e is 50, key is test_row_0/B:col10/1733145525671/Put/seqid=0 2024-12-02T13:18:46,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742187_1363 (size=12001) 2024-12-02T13:18:46,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:46,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145586344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145586374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145586383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145586383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145586383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2b096612b27e4f29a6046faba444e63e 2024-12-02T13:18:46,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a9893af2d24e4a42a9a4c2449e22433d is 50, key is test_row_0/C:col10/1733145525671/Put/seqid=0 2024-12-02T13:18:46,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:46,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742188_1364 (size=12001) 2024-12-02T13:18:46,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a9893af2d24e4a42a9a4c2449e22433d 2024-12-02T13:18:46,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/d554de96a5dd4a0d90cda44aeecac84c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c 2024-12-02T13:18:46,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T13:18:46,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2b096612b27e4f29a6046faba444e63e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e 2024-12-02T13:18:46,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T13:18:46,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a9893af2d24e4a42a9a4c2449e22433d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d 2024-12-02T13:18:46,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T13:18:46,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2fbac5b7f28671050128676c4e02d4f1 in 938ms, sequenceid=15, compaction requested=false 2024-12-02T13:18:46,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:46,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:46,736 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:46,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:18:46,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020743b5e33c08459693c1f9eca534c1ae_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145525742/Put/seqid=0 2024-12-02T13:18:46,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742189_1365 (size=12154) 2024-12-02T13:18:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:46,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:46,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145586893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145586894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145586894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145586896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:46,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:46,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145586899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145587014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145587014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:47,158 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020743b5e33c08459693c1f9eca534c1ae_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020743b5e33c08459693c1f9eca534c1ae_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:47,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0bbbf0defb9843e7b30387632646eaf1, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:47,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0bbbf0defb9843e7b30387632646eaf1 is 175, key is test_row_0/A:col10/1733145525742/Put/seqid=0 2024-12-02T13:18:47,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742190_1366 (size=30955) 2024-12-02T13:18:47,172 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0bbbf0defb9843e7b30387632646eaf1 2024-12-02T13:18:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 is 50, key is test_row_0/B:col10/1733145525742/Put/seqid=0 2024-12-02T13:18:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742191_1367 (size=12001) 2024-12-02T13:18:47,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145587234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145587235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,248 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:18:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145587544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145587544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,611 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 2024-12-02T13:18:47,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/b3ac63a39cd64ad996ed706e43621d95 is 50, key is test_row_0/C:col10/1733145525742/Put/seqid=0 2024-12-02T13:18:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742192_1368 (size=12001) 2024-12-02T13:18:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145587905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,911 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145587914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145587914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:47,920 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:47,920 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:48,039 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/b3ac63a39cd64ad996ed706e43621d95 2024-12-02T13:18:48,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0bbbf0defb9843e7b30387632646eaf1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1 2024-12-02T13:18:48,056 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T13:18:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 2024-12-02T13:18:48,065 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T13:18:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145588053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:48,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145588054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:48,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/b3ac63a39cd64ad996ed706e43621d95 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95 2024-12-02T13:18:48,079 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T13:18:48,080 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 2fbac5b7f28671050128676c4e02d4f1 in 1344ms, sequenceid=49, compaction requested=false 2024-12-02T13:18:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:48,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T13:18:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T13:18:48,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T13:18:48,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4180 sec 2024-12-02T13:18:48,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4290 sec 2024-12-02T13:18:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:49,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:49,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:49,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020b28dfb6bab14cd293252280d97cc4d9_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:49,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742193_1369 (size=14594) 2024-12-02T13:18:49,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145589136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145589137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145589253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145589254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145589464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145589474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:49,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020b28dfb6bab14cd293252280d97cc4d9_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020b28dfb6bab14cd293252280d97cc4d9_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:49,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af56c8412f8e42759485307f20cdd43f, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:49,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af56c8412f8e42759485307f20cdd43f is 175, key is test_row_0/A:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:49,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742194_1370 (size=39549) 2024-12-02T13:18:49,532 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af56c8412f8e42759485307f20cdd43f 2024-12-02T13:18:49,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2f7132522cff4ff598058e76c123377d is 50, key is test_row_0/B:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:49,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742195_1371 (size=12001) 2024-12-02T13:18:49,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145589774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145589793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T13:18:49,824 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:49,826 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-02T13:18:49,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:49,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:49,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145589914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,920 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145589934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145589934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:49,942 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,942 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:49,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T13:18:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:49,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:49,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:49,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:49,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2f7132522cff4ff598058e76c123377d 2024-12-02T13:18:49,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c2a177bc19304dbbabc063d2df608a6c is 50, key is test_row_0/C:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:50,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742196_1372 (size=12001) 2024-12-02T13:18:50,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T13:18:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:50,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:50,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T13:18:50,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:50,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145590294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:50,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145590314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:50,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:50,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c2a177bc19304dbbabc063d2df608a6c 2024-12-02T13:18:50,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af56c8412f8e42759485307f20cdd43f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f 2024-12-02T13:18:50,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f, entries=200, sequenceid=62, filesize=38.6 K 2024-12-02T13:18:50,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/2f7132522cff4ff598058e76c123377d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d 2024-12-02T13:18:50,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T13:18:50,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c2a177bc19304dbbabc063d2df608a6c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c 2024-12-02T13:18:50,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c, entries=150, sequenceid=62, filesize=11.7 K 2024-12-02T13:18:50,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2fbac5b7f28671050128676c4e02d4f1 in 1350ms, sequenceid=62, compaction requested=true 2024-12-02T13:18:50,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:50,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:50,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:50,426 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:50,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:50,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:50,426 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:50,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:50,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:50,430 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:50,430 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:50,430 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:18:50,430 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:18:50,430 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,430 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,430 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.2 K 2024-12-02T13:18:50,430 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=99.1 K 2024-12-02T13:18:50,430 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,430 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f] 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d554de96a5dd4a0d90cda44aeecac84c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145525665 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b096612b27e4f29a6046faba444e63e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145525665 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bbbf0defb9843e7b30387632646eaf1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145525742 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bf66718b0d124d0e9a8bf6ae0d7b2e09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145525742 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting af56c8412f8e42759485307f20cdd43f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:50,437 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f7132522cff4ff598058e76c123377d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:50,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,444 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:50,459 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:50,461 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/15440fd98b4a43bda0dfd36a3451634e is 50, key is test_row_0/B:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:50,470 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022bc8bbceeb6143ffb4b8f8d5fcd1b296_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145529135/Put/seqid=0 2024-12-02T13:18:50,478 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202fb2c61b9bbe04012b76b7c114c986322_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:50,479 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202fb2c61b9bbe04012b76b7c114c986322_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:50,480 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fb2c61b9bbe04012b76b7c114c986322_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742197_1373 (size=12104) 2024-12-02T13:18:50,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742198_1374 (size=12154) 2024-12-02T13:18:50,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742199_1375 (size=4469) 2024-12-02T13:18:50,894 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/15440fd98b4a43bda0dfd36a3451634e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/15440fd98b4a43bda0dfd36a3451634e 2024-12-02T13:18:50,898 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 15440fd98b4a43bda0dfd36a3451634e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:50,898 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:50,898 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145530426; duration=0sec 2024-12-02T13:18:50,898 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:50,898 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:18:50,898 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:50,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:50,901 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:50,901 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:18:50,901 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:50,901 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.2 K 2024-12-02T13:18:50,901 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a9893af2d24e4a42a9a4c2449e22433d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145525665 2024-12-02T13:18:50,902 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ac63a39cd64ad996ed706e43621d95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145525742 2024-12-02T13:18:50,902 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a177bc19304dbbabc063d2df608a6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:50,904 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:50,905 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022bc8bbceeb6143ffb4b8f8d5fcd1b296_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022bc8bbceeb6143ffb4b8f8d5fcd1b296_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:50,905 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/b602d9da2a554a5e8c81753991600e63 is 175, key is test_row_0/A:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:50,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/019bedd0fd7e4389bb7a2b692cc9d75b, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:50,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/019bedd0fd7e4389bb7a2b692cc9d75b is 175, key is test_row_0/A:col10/1733145529135/Put/seqid=0 2024-12-02T13:18:50,915 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:50,916 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/de1fd5de99ea4a32a9522d7712906cb7 is 50, key is test_row_0/C:col10/1733145526882/Put/seqid=0 2024-12-02T13:18:50,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742201_1377 (size=30955) 2024-12-02T13:18:50,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742200_1376 (size=31058) 2024-12-02T13:18:50,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742202_1378 (size=12104) 2024-12-02T13:18:50,939 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/de1fd5de99ea4a32a9522d7712906cb7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/de1fd5de99ea4a32a9522d7712906cb7 2024-12-02T13:18:50,954 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into de1fd5de99ea4a32a9522d7712906cb7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:50,954 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:50,954 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145530426; duration=0sec 2024-12-02T13:18:50,954 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:50,954 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:18:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/019bedd0fd7e4389bb7a2b692cc9d75b 2024-12-02T13:18:51,327 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/b602d9da2a554a5e8c81753991600e63 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63 2024-12-02T13:18:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e850005b4f9b4ced9c55f7b7db1f24d6 is 50, key is test_row_0/B:col10/1733145529135/Put/seqid=0 2024-12-02T13:18:51,334 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into b602d9da2a554a5e8c81753991600e63(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:51,334 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:51,334 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145530426; duration=0sec 2024-12-02T13:18:51,334 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:51,334 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:18:51,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742203_1379 (size=12001) 2024-12-02T13:18:51,346 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e850005b4f9b4ced9c55f7b7db1f24d6 2024-12-02T13:18:51,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/635cc4ba8ee146cf994a8b53dce19943 is 50, key is test_row_0/C:col10/1733145529135/Put/seqid=0 2024-12-02T13:18:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742204_1380 (size=12001) 2024-12-02T13:18:51,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145591365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145591369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145591484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145591484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145591694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145591704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:51,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:51,764 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/635cc4ba8ee146cf994a8b53dce19943 2024-12-02T13:18:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/019bedd0fd7e4389bb7a2b692cc9d75b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b 2024-12-02T13:18:51,773 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b, entries=150, sequenceid=81, filesize=30.2 K 2024-12-02T13:18:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e850005b4f9b4ced9c55f7b7db1f24d6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6 2024-12-02T13:18:51,779 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6, entries=150, sequenceid=81, filesize=11.7 K 2024-12-02T13:18:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/635cc4ba8ee146cf994a8b53dce19943 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943 2024-12-02T13:18:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943, entries=150, sequenceid=81, filesize=11.7 K 2024-12-02T13:18:51,785 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2fbac5b7f28671050128676c4e02d4f1 in 1341ms, sequenceid=81, compaction requested=false 2024-12-02T13:18:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:51,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-02T13:18:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-02T13:18:51,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T13:18:51,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-12-02T13:18:51,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.9640 sec 2024-12-02T13:18:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T13:18:51,984 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:51,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-02T13:18:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:51,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:51,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:51,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:52,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:52,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029f5ed0b208074ccf8e97725da2897081_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742205_1381 (size=14594) 2024-12-02T13:18:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145592111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145592118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145592234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145592244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:52,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145592443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145592454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:52,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029f5ed0b208074ccf8e97725da2897081_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029f5ed0b208074ccf8e97725da2897081_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:52,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/c49454930fcf4311a370619881bd9111, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:52,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/c49454930fcf4311a370619881bd9111 is 175, key is test_row_0/A:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:52,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742206_1382 (size=39549) 2024-12-02T13:18:52,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:52,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145592757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145592764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:52,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:52,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/c49454930fcf4311a370619881bd9111 2024-12-02T13:18:52,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:52,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:52,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e3f7e363d0a54ebc92ef478f3d829118 is 50, key is test_row_0/B:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:52,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742207_1383 (size=12001) 2024-12-02T13:18:52,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e3f7e363d0a54ebc92ef478f3d829118 2024-12-02T13:18:52,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/520d1d5c42bc42728ce4686e5125403d is 50, key is test_row_0/C:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742208_1384 (size=12001) 2024-12-02T13:18:52,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/520d1d5c42bc42728ce4686e5125403d 2024-12-02T13:18:52,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/c49454930fcf4311a370619881bd9111 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111 2024-12-02T13:18:52,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111, entries=200, sequenceid=98, filesize=38.6 K 2024-12-02T13:18:52,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e3f7e363d0a54ebc92ef478f3d829118 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118 2024-12-02T13:18:53,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118, entries=150, sequenceid=98, filesize=11.7 K 2024-12-02T13:18:53,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/520d1d5c42bc42728ce4686e5125403d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d 2024-12-02T13:18:53,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d, entries=150, sequenceid=98, filesize=11.7 K 2024-12-02T13:18:53,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2fbac5b7f28671050128676c4e02d4f1 in 1005ms, sequenceid=98, compaction requested=true 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:53,013 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:18:53,013 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:53,014 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:53,014 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:18:53,014 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,014 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=99.2 K 2024-12-02T13:18:53,015 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,015 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111] 2024-12-02T13:18:53,015 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b602d9da2a554a5e8c81753991600e63, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:53,015 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019bedd0fd7e4389bb7a2b692cc9d75b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733145529115 2024-12-02T13:18:53,016 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:53,016 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:18:53,016 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,016 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/15440fd98b4a43bda0dfd36a3451634e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.3 K 2024-12-02T13:18:53,016 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49454930fcf4311a370619881bd9111, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:53,016 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 15440fd98b4a43bda0dfd36a3451634e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:53,017 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e850005b4f9b4ced9c55f7b7db1f24d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733145529115 2024-12-02T13:18:53,017 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f7e363d0a54ebc92ef478f3d829118, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:53,022 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:53,031 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:53,031 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/fae691c7647c459f954edd32b2bbbc13 is 50, key is test_row_0/B:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:53,040 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120269f1f4376f324e9cb6d9e951bb6ea3cf_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:53,041 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120269f1f4376f324e9cb6d9e951bb6ea3cf_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:53,041 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120269f1f4376f324e9cb6d9e951bb6ea3cf_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:53,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742209_1385 (size=12207) 2024-12-02T13:18:53,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T13:18:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,064 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:18:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:53,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742210_1386 (size=4469) 2024-12-02T13:18:53,090 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#326 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:53,091 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a80025741f7748aa835e25da62442f98 is 175, key is test_row_0/A:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eebb31580c1740f1a9cb72be89f31e29_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145532089/Put/seqid=0 2024-12-02T13:18:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742211_1387 (size=31161) 2024-12-02T13:18:53,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742212_1388 (size=12154) 2024-12-02T13:18:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:53,130 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eebb31580c1740f1a9cb72be89f31e29_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eebb31580c1740f1a9cb72be89f31e29_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:53,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4b2ca8d68ad14f35a4bd8d9349c5f588, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:53,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4b2ca8d68ad14f35a4bd8d9349c5f588 is 175, key is test_row_0/A:col10/1733145532089/Put/seqid=0 2024-12-02T13:18:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:53,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742213_1389 (size=30955) 2024-12-02T13:18:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:53,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145593326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145593328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145593444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145593444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,468 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/fae691c7647c459f954edd32b2bbbc13 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/fae691c7647c459f954edd32b2bbbc13 2024-12-02T13:18:53,480 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into fae691c7647c459f954edd32b2bbbc13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:53,480 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:53,480 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145533013; duration=0sec 2024-12-02T13:18:53,480 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:53,480 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:18:53,480 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:53,481 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:53,481 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:18:53,482 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,482 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/de1fd5de99ea4a32a9522d7712906cb7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.3 K 2024-12-02T13:18:53,482 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting de1fd5de99ea4a32a9522d7712906cb7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733145526882 2024-12-02T13:18:53,482 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 635cc4ba8ee146cf994a8b53dce19943, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733145529115 2024-12-02T13:18:53,482 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 520d1d5c42bc42728ce4686e5125403d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:53,490 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:53,491 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/625f1d0068ea4bc99cc4bc96fc7e63f1 is 50, key is test_row_0/C:col10/1733145531339/Put/seqid=0 2024-12-02T13:18:53,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742214_1390 (size=12207) 2024-12-02T13:18:53,517 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a80025741f7748aa835e25da62442f98 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98 2024-12-02T13:18:53,522 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into a80025741f7748aa835e25da62442f98(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:53,522 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:53,522 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145533013; duration=0sec 2024-12-02T13:18:53,522 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:53,522 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:18:53,539 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4b2ca8d68ad14f35a4bd8d9349c5f588 2024-12-02T13:18:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e1e1bf2e974c4cf5a8268d5b877205e0 is 50, key is test_row_0/B:col10/1733145532089/Put/seqid=0 2024-12-02T13:18:53,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742215_1391 (size=12001) 2024-12-02T13:18:53,585 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e1e1bf2e974c4cf5a8268d5b877205e0 2024-12-02T13:18:53,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/67cf127e246f4142abe98df31db6c4ae is 50, key is test_row_0/C:col10/1733145532089/Put/seqid=0 2024-12-02T13:18:53,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742216_1392 (size=12001) 2024-12-02T13:18:53,608 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/67cf127e246f4142abe98df31db6c4ae 2024-12-02T13:18:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4b2ca8d68ad14f35a4bd8d9349c5f588 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588 2024-12-02T13:18:53,618 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588, entries=150, sequenceid=116, filesize=30.2 K 2024-12-02T13:18:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/e1e1bf2e974c4cf5a8268d5b877205e0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0 2024-12-02T13:18:53,625 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0, entries=150, sequenceid=116, filesize=11.7 K 2024-12-02T13:18:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/67cf127e246f4142abe98df31db6c4ae as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae 2024-12-02T13:18:53,629 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae, entries=150, sequenceid=116, filesize=11.7 K 2024-12-02T13:18:53,630 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2fbac5b7f28671050128676c4e02d4f1 in 566ms, sequenceid=116, compaction requested=false 2024-12-02T13:18:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-02T13:18:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-02T13:18:53,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T13:18:53,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-12-02T13:18:53,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6490 sec 2024-12-02T13:18:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:53,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:53,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023cd0fe4643b54c1fb7568f3d18ef4445_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742217_1393 (size=12254) 2024-12-02T13:18:53,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145593719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145593721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145593834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145593843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,916 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/625f1d0068ea4bc99cc4bc96fc7e63f1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/625f1d0068ea4bc99cc4bc96fc7e63f1 2024-12-02T13:18:53,922 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into 625f1d0068ea4bc99cc4bc96fc7e63f1(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:53,922 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:53,922 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145533013; duration=0sec 2024-12-02T13:18:53,922 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:53,922 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:18:53,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145593955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,964 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145593964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145593964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:53,970 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:53,971 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:18:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145594054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145594064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,086 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:54,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023cd0fe4643b54c1fb7568f3d18ef4445_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd0fe4643b54c1fb7568f3d18ef4445_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:54,099 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4a5b1c5f1f6142479e02e2e1a1fa2e62, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 is 175, key is test_row_0/A:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742218_1394 (size=31055) 2024-12-02T13:18:54,112 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 2024-12-02T13:18:54,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/732e68207dfa4f54a73a4a4a9c69e0d3 is 50, key is test_row_0/B:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742219_1395 (size=12101) 2024-12-02T13:18:54,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/732e68207dfa4f54a73a4a4a9c69e0d3 2024-12-02T13:18:54,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/e188bf26862145d39d15d97b37531832 is 50, key is test_row_0/C:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:54,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742220_1396 (size=12101) 2024-12-02T13:18:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T13:18:54,144 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:54,145 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-02T13:18:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:54,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:54,150 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:54,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:54,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T13:18:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145594364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145594384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T13:18:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:54,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/e188bf26862145d39d15d97b37531832 2024-12-02T13:18:54,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 2024-12-02T13:18:54,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62, entries=150, sequenceid=133, filesize=30.3 K 2024-12-02T13:18:54,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/732e68207dfa4f54a73a4a4a9c69e0d3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3 2024-12-02T13:18:54,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3, entries=150, sequenceid=133, filesize=11.8 K 2024-12-02T13:18:54,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/e188bf26862145d39d15d97b37531832 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832 2024-12-02T13:18:54,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832, entries=150, sequenceid=133, filesize=11.8 K 2024-12-02T13:18:54,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2fbac5b7f28671050128676c4e02d4f1 in 906ms, sequenceid=133, compaction requested=true 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T13:18:54,563 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:54,563 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:18:54,565 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,565 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/625f1d0068ea4bc99cc4bc96fc7e63f1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.5 K 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:18:54,565 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 625f1d0068ea4bc99cc4bc96fc7e63f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:54,565 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=91.0 K 2024-12-02T13:18:54,565 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,565 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62] 2024-12-02T13:18:54,566 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 67cf127e246f4142abe98df31db6c4ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733145532081 2024-12-02T13:18:54,566 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80025741f7748aa835e25da62442f98, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:54,566 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2ca8d68ad14f35a4bd8d9349c5f588, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733145532081 2024-12-02T13:18:54,566 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e188bf26862145d39d15d97b37531832, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:54,566 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5b1c5f1f6142479e02e2e1a1fa2e62, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:54,574 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:54,574 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/95d0564be1f94d509c71cb5ba3152765 is 50, key is test_row_0/C:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:54,582 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,585 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a872b049d8d44bf39e61af247e58f70a_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,587 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a872b049d8d44bf39e61af247e58f70a_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,587 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a872b049d8d44bf39e61af247e58f70a_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742222_1398 (size=4469) 2024-12-02T13:18:54,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742221_1397 (size=12409) 2024-12-02T13:18:54,596 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#336 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:54,596 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9 is 175, key is test_row_0/A:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:54,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T13:18:54,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:54,610 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:54,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742223_1399 (size=31363) 2024-12-02T13:18:54,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120267cfdd7c5cc6476bb10237bfab408743_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145533706/Put/seqid=0 2024-12-02T13:18:54,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742224_1400 (size=12304) 2024-12-02T13:18:54,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:54,633 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120267cfdd7c5cc6476bb10237bfab408743_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120267cfdd7c5cc6476bb10237bfab408743_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8ead1ea3fb464583948ad761045fd792, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8ead1ea3fb464583948ad761045fd792 is 175, key is test_row_0/A:col10/1733145533706/Put/seqid=0 2024-12-02T13:18:54,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742225_1401 (size=31105) 2024-12-02T13:18:54,643 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8ead1ea3fb464583948ad761045fd792 2024-12-02T13:18:54,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/ca1d10a95a9549448435689fbb6d4170 is 50, key is test_row_0/B:col10/1733145533706/Put/seqid=0 2024-12-02T13:18:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742226_1402 (size=12151) 2024-12-02T13:18:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:54,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:54,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145594952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145594954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:54,998 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/95d0564be1f94d509c71cb5ba3152765 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/95d0564be1f94d509c71cb5ba3152765 2024-12-02T13:18:55,002 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into 95d0564be1f94d509c71cb5ba3152765(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:55,002 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:55,002 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145534563; duration=0sec 2024-12-02T13:18:55,002 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:55,002 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:18:55,002 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:55,003 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:55,003 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:18:55,003 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:55,003 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/fae691c7647c459f954edd32b2bbbc13, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.5 K 2024-12-02T13:18:55,003 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fae691c7647c459f954edd32b2bbbc13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733145531339 2024-12-02T13:18:55,003 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e1bf2e974c4cf5a8268d5b877205e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733145532081 2024-12-02T13:18:55,004 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 732e68207dfa4f54a73a4a4a9c69e0d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:55,009 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:55,009 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b2f5d28b66dd4637b1c764e14ee4bce3 is 50, key is test_row_0/B:col10/1733145533326/Put/seqid=0 2024-12-02T13:18:55,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742227_1403 (size=12409) 2024-12-02T13:18:55,024 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9 2024-12-02T13:18:55,028 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into 4c3e85cd0c6a4cf8b1ceb220c7776ed9(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:55,028 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:55,028 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145534563; duration=0sec 2024-12-02T13:18:55,028 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:55,028 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:18:55,054 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/ca1d10a95a9549448435689fbb6d4170 2024-12-02T13:18:55,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/314df6b2f10b4666b965312185083974 is 50, key is test_row_0/C:col10/1733145533706/Put/seqid=0 2024-12-02T13:18:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145595063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145595063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742228_1404 (size=12151) 2024-12-02T13:18:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145595274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145595274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:55,417 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b2f5d28b66dd4637b1c764e14ee4bce3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b2f5d28b66dd4637b1c764e14ee4bce3 2024-12-02T13:18:55,424 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into b2f5d28b66dd4637b1c764e14ee4bce3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:55,424 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:55,424 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145534563; duration=0sec 2024-12-02T13:18:55,424 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:55,425 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:18:55,474 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/314df6b2f10b4666b965312185083974 2024-12-02T13:18:55,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8ead1ea3fb464583948ad761045fd792 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792 2024-12-02T13:18:55,481 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792, entries=150, sequenceid=151, filesize=30.4 K 2024-12-02T13:18:55,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/ca1d10a95a9549448435689fbb6d4170 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170 2024-12-02T13:18:55,484 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170, entries=150, sequenceid=151, filesize=11.9 K 2024-12-02T13:18:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/314df6b2f10b4666b965312185083974 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974 2024-12-02T13:18:55,487 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974, entries=150, sequenceid=151, filesize=11.9 K 2024-12-02T13:18:55,488 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2fbac5b7f28671050128676c4e02d4f1 in 878ms, sequenceid=151, compaction requested=false 2024-12-02T13:18:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:55,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-02T13:18:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-02T13:18:55,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-02T13:18:55,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-12-02T13:18:55,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3460 sec 2024-12-02T13:18:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:55,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ca3175e7db74909ae9fb41c56a75250_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:55,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742229_1405 (size=12304) 2024-12-02T13:18:55,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:55,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ca3175e7db74909ae9fb41c56a75250_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ca3175e7db74909ae9fb41c56a75250_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:55,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0acb8a25500a4ca8b45b628540c72555, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:55,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0acb8a25500a4ca8b45b628540c72555 is 175, key is test_row_0/A:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742230_1406 (size=31105) 2024-12-02T13:18:55,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145595635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145595641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145595744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145595754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145595953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:55,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145595964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0acb8a25500a4ca8b45b628540c72555 2024-12-02T13:18:56,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/983e8d9186d141e6a207b5645a073dbf is 50, key is test_row_0/B:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:56,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742231_1407 (size=12151) 2024-12-02T13:18:56,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/983e8d9186d141e6a207b5645a073dbf 2024-12-02T13:18:56,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/2b68d22299d749618718558c4bc2d7f2 is 50, key is test_row_0/C:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:56,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742232_1408 (size=12151) 2024-12-02T13:18:56,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145596264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145596274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T13:18:56,293 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:56,294 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-02T13:18:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:56,295 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:56,296 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:56,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:56,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/2b68d22299d749618718558c4bc2d7f2 2024-12-02T13:18:56,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0acb8a25500a4ca8b45b628540c72555 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555 2024-12-02T13:18:56,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555, entries=150, sequenceid=170, filesize=30.4 K 2024-12-02T13:18:56,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/983e8d9186d141e6a207b5645a073dbf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf 2024-12-02T13:18:56,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-02T13:18:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:18:56,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T13:18:56,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/2b68d22299d749618718558c4bc2d7f2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2 2024-12-02T13:18:56,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T13:18:56,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2fbac5b7f28671050128676c4e02d4f1 in 868ms, sequenceid=170, compaction requested=true 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:56,455 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:56,455 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:18:56,456 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,456 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,456 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b2f5d28b66dd4637b1c764e14ee4bce3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.9 K 2024-12-02T13:18:56,456 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=91.4 K 2024-12-02T13:18:56,456 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555] 2024-12-02T13:18:56,456 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f5d28b66dd4637b1c764e14ee4bce3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:56,457 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3e85cd0c6a4cf8b1ceb220c7776ed9, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:56,457 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1d10a95a9549448435689fbb6d4170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733145533694 2024-12-02T13:18:56,457 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ead1ea3fb464583948ad761045fd792, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733145533694 2024-12-02T13:18:56,457 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 983e8d9186d141e6a207b5645a073dbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:56,457 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0acb8a25500a4ca8b45b628540c72555, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:56,463 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:56,464 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:56,464 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/66197c65f02e4045b2733cdd5273d426 is 50, key is test_row_0/B:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:56,465 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412027697a8a9695247d991f06973859c6789_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:56,467 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412027697a8a9695247d991f06973859c6789_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:56,467 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027697a8a9695247d991f06973859c6789_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:56,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742234_1410 (size=4469) 2024-12-02T13:18:56,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742233_1409 (size=12561) 2024-12-02T13:18:56,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,601 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:56,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f221a8409fd84a4b9e28563f845bb5f4_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145535618/Put/seqid=0 2024-12-02T13:18:56,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742235_1411 (size=12304) 2024-12-02T13:18:56,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:56,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145596816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145596819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,879 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#345 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:56,880 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/997be22454ad4a76b7b5cc4d3a65fec3 is 175, key is test_row_0/A:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742236_1412 (size=31515) 2024-12-02T13:18:56,884 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/66197c65f02e4045b2733cdd5273d426 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/66197c65f02e4045b2733cdd5273d426 2024-12-02T13:18:56,888 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 66197c65f02e4045b2733cdd5273d426(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:56,888 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145536455; duration=0sec 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:56,888 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:18:56,889 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:56,889 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/95d0564be1f94d509c71cb5ba3152765, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=35.9 K 2024-12-02T13:18:56,889 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d0564be1f94d509c71cb5ba3152765, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733145533322 2024-12-02T13:18:56,889 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 314df6b2f10b4666b965312185083974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733145533694 2024-12-02T13:18:56,889 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b68d22299d749618718558c4bc2d7f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:56,895 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:56,895 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/e6c936adbd7a4063aaf826d5fab218b1 is 50, key is test_row_0/C:col10/1733145534926/Put/seqid=0 2024-12-02T13:18:56,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742237_1413 (size=12561) 2024-12-02T13:18:56,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145596933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145596933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:57,013 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f221a8409fd84a4b9e28563f845bb5f4_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f221a8409fd84a4b9e28563f845bb5f4_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/caecbe24e4844b89a88dd4110570e885, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/caecbe24e4844b89a88dd4110570e885 is 175, key is test_row_0/A:col10/1733145535618/Put/seqid=0 2024-12-02T13:18:57,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742238_1414 (size=31105) 2024-12-02T13:18:57,017 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/caecbe24e4844b89a88dd4110570e885 2024-12-02T13:18:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b789afdf9cee42deb654d91cecfd82e6 is 50, key is test_row_0/B:col10/1733145535618/Put/seqid=0 2024-12-02T13:18:57,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742239_1415 (size=12151) 2024-12-02T13:18:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b789afdf9cee42deb654d91cecfd82e6 2024-12-02T13:18:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/29a77b5219634dfcb34798a06361055d is 50, key is test_row_0/C:col10/1733145535618/Put/seqid=0 2024-12-02T13:18:57,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742240_1416 (size=12151) 2024-12-02T13:18:57,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145597143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145597143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,287 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/997be22454ad4a76b7b5cc4d3a65fec3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3 2024-12-02T13:18:57,291 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into 997be22454ad4a76b7b5cc4d3a65fec3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:57,291 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:57,291 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145536455; duration=0sec 2024-12-02T13:18:57,291 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:57,291 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:18:57,302 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/e6c936adbd7a4063aaf826d5fab218b1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e6c936adbd7a4063aaf826d5fab218b1 2024-12-02T13:18:57,305 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into e6c936adbd7a4063aaf826d5fab218b1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:57,305 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:57,305 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145536455; duration=0sec 2024-12-02T13:18:57,305 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:57,305 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:18:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:57,435 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/29a77b5219634dfcb34798a06361055d 2024-12-02T13:18:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/caecbe24e4844b89a88dd4110570e885 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885 2024-12-02T13:18:57,441 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885, entries=150, sequenceid=186, filesize=30.4 K 2024-12-02T13:18:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b789afdf9cee42deb654d91cecfd82e6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6 2024-12-02T13:18:57,444 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6, entries=150, sequenceid=186, filesize=11.9 K 2024-12-02T13:18:57,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/29a77b5219634dfcb34798a06361055d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d 2024-12-02T13:18:57,448 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d, entries=150, sequenceid=186, filesize=11.9 K 2024-12-02T13:18:57,448 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2fbac5b7f28671050128676c4e02d4f1 in 847ms, sequenceid=186, compaction requested=false 2024-12-02T13:18:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-02T13:18:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-02T13:18:57,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-02T13:18:57,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1540 sec 2024-12-02T13:18:57,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1570 sec 2024-12-02T13:18:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:57,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:57,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ad7e1f6205243778f94cd20cb1d0554_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:57,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742241_1417 (size=17284) 2024-12-02T13:18:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145597496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145597500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145597604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145597614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145597814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145597824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:57,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:57,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ad7e1f6205243778f94cd20cb1d0554_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ad7e1f6205243778f94cd20cb1d0554_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:57,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8b76d8dd75314a19a4006a97a7c3b0dd, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:57,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8b76d8dd75314a19a4006a97a7c3b0dd is 175, key is test_row_0/A:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:57,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742242_1418 (size=48389) 2024-12-02T13:18:57,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8b76d8dd75314a19a4006a97a7c3b0dd 2024-12-02T13:18:57,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/6af6620be6434056b77eb147cbed89ad is 50, key is test_row_0/B:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742243_1419 (size=12151) 2024-12-02T13:18:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145598134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145598134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:58,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/6af6620be6434056b77eb147cbed89ad 2024-12-02T13:18:58,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/5622d0c18da849749d94fe468376643b is 50, key is test_row_0/C:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:58,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742244_1420 (size=12151) 2024-12-02T13:18:58,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/5622d0c18da849749d94fe468376643b 2024-12-02T13:18:58,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/8b76d8dd75314a19a4006a97a7c3b0dd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd 2024-12-02T13:18:58,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd, entries=250, sequenceid=207, filesize=47.3 K 2024-12-02T13:18:58,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/6af6620be6434056b77eb147cbed89ad as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad 2024-12-02T13:18:58,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad, entries=150, sequenceid=207, filesize=11.9 K 2024-12-02T13:18:58,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/5622d0c18da849749d94fe468376643b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b 2024-12-02T13:18:58,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b, entries=150, sequenceid=207, filesize=11.9 K 2024-12-02T13:18:58,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2fbac5b7f28671050128676c4e02d4f1 in 858ms, sequenceid=207, compaction requested=true 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:58,317 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:58,317 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:18:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:58,318 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:58,318 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:58,318 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:18:58,318 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:18:58,318 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:58,319 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:58,319 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/66197c65f02e4045b2733cdd5273d426, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.0 K 2024-12-02T13:18:58,319 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=108.4 K 2024-12-02T13:18:58,319 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd] 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 66197c65f02e4045b2733cdd5273d426, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 997be22454ad4a76b7b5cc4d3a65fec3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b789afdf9cee42deb654d91cecfd82e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733145535618 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting caecbe24e4844b89a88dd4110570e885, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733145535618 2024-12-02T13:18:58,319 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af6620be6434056b77eb147cbed89ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536818 2024-12-02T13:18:58,320 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b76d8dd75314a19a4006a97a7c3b0dd, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536791 2024-12-02T13:18:58,323 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:58,324 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:58,325 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/1c6d3af48b5d49bba5bb374dd2cffd12 is 50, key is test_row_0/B:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:58,326 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ce525d7da659401a9b7f1a33e2abbe2f_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:58,327 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ce525d7da659401a9b7f1a33e2abbe2f_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:58,328 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ce525d7da659401a9b7f1a33e2abbe2f_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:58,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742245_1421 (size=12663) 2024-12-02T13:18:58,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742246_1422 (size=4469) 2024-12-02T13:18:58,337 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/1c6d3af48b5d49bba5bb374dd2cffd12 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/1c6d3af48b5d49bba5bb374dd2cffd12 2024-12-02T13:18:58,341 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 1c6d3af48b5d49bba5bb374dd2cffd12(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:58,341 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:58,341 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145538317; duration=0sec 2024-12-02T13:18:58,341 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:18:58,341 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:18:58,341 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:18:58,342 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:18:58,342 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:18:58,342 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:58,342 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e6c936adbd7a4063aaf826d5fab218b1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.0 K 2024-12-02T13:18:58,342 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c936adbd7a4063aaf826d5fab218b1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145534900 2024-12-02T13:18:58,343 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a77b5219634dfcb34798a06361055d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733145535618 2024-12-02T13:18:58,343 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5622d0c18da849749d94fe468376643b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536818 2024-12-02T13:18:58,348 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:58,348 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/1e42f3391b714974931f10a62cd76189 is 50, key is test_row_0/C:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:58,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742247_1423 (size=12663) 2024-12-02T13:18:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T13:18:58,444 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:18:58,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:18:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-02T13:18:58,446 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:18:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:18:58,446 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:18:58,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:18:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:18:58,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:58,598 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:18:58,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:18:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120283e167add59b4ceebf0bb3a3c8b926ea_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145537484/Put/seqid=0 2024-12-02T13:18:58,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742248_1424 (size=12304) 2024-12-02T13:18:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:58,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:18:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145598693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:58,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145598702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:58,732 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:18:58,732 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ffbd3ba1ed35431190a826585c301d40 is 175, key is test_row_0/A:col10/1733145537458/Put/seqid=0 2024-12-02T13:18:58,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742249_1425 (size=31617) 2024-12-02T13:18:58,739 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ffbd3ba1ed35431190a826585c301d40 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40 2024-12-02T13:18:58,743 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into ffbd3ba1ed35431190a826585c301d40(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:58,743 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:58,743 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145538317; duration=0sec 2024-12-02T13:18:58,743 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:58,743 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:18:58,756 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/1e42f3391b714974931f10a62cd76189 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/1e42f3391b714974931f10a62cd76189 2024-12-02T13:18:58,759 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into 1e42f3391b714974931f10a62cd76189(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:18:58,759 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:58,759 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145538317; duration=0sec 2024-12-02T13:18:58,759 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:18:58,759 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:18:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:18:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145598814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145598814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:18:59,010 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120283e167add59b4ceebf0bb3a3c8b926ea_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120283e167add59b4ceebf0bb3a3c8b926ea_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:18:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/3445ae8db11d497e9160d732db727b43, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:18:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/3445ae8db11d497e9160d732db727b43 is 175, key is test_row_0/A:col10/1733145537484/Put/seqid=0 2024-12-02T13:18:59,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742250_1426 (size=31105) 2024-12-02T13:18:59,015 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/3445ae8db11d497e9160d732db727b43 2024-12-02T13:18:59,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/220df5d32fdd409eb08481e97b16c69c is 50, key is test_row_0/B:col10/1733145537484/Put/seqid=0 2024-12-02T13:18:59,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742251_1427 (size=12151) 2024-12-02T13:18:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145599023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145599023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:18:59,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145599334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145599334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,423 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/220df5d32fdd409eb08481e97b16c69c 2024-12-02T13:18:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a15b2b6dd890469b80c45c48f7906731 is 50, key is test_row_0/C:col10/1733145537484/Put/seqid=0 2024-12-02T13:18:59,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742252_1428 (size=12151) 2024-12-02T13:18:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:18:59,839 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a15b2b6dd890469b80c45c48f7906731 2024-12-02T13:18:59,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/3445ae8db11d497e9160d732db727b43 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43 2024-12-02T13:18:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43, entries=150, sequenceid=222, filesize=30.4 K 2024-12-02T13:18:59,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/220df5d32fdd409eb08481e97b16c69c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c 2024-12-02T13:18:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145599843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145599843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:18:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:18:59,848 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c, entries=150, sequenceid=222, filesize=11.9 K 2024-12-02T13:18:59,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/a15b2b6dd890469b80c45c48f7906731 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731 2024-12-02T13:18:59,851 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731, entries=150, sequenceid=222, filesize=11.9 K 2024-12-02T13:18:59,852 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2fbac5b7f28671050128676c4e02d4f1 in 1254ms, sequenceid=222, compaction requested=false 2024-12-02T13:18:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:18:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:18:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-02T13:18:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-02T13:18:59,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T13:18:59,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-12-02T13:18:59,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4120 sec 2024-12-02T13:19:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-02T13:19:00,594 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:00,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-02T13:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:00,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:00,597 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:00,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:00,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-02T13:19:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:00,750 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:00,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ace78c6a855484087d608ea2e199fcb_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145538674/Put/seqid=0 2024-12-02T13:19:00,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742253_1429 (size=12304) 2024-12-02T13:19:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:00,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:00,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:00,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145600884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145600889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145601004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145601013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:01,162 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026ace78c6a855484087d608ea2e199fcb_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ace78c6a855484087d608ea2e199fcb_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a885b0230d9a4e66b0695280657ddac2, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a885b0230d9a4e66b0695280657ddac2 is 175, key is test_row_0/A:col10/1733145538674/Put/seqid=0 2024-12-02T13:19:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742254_1430 (size=31105) 2024-12-02T13:19:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145601224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145601234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145601533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145601544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:01,571 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a885b0230d9a4e66b0695280657ddac2 2024-12-02T13:19:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/a4d8ccedc74b43c2a995fc9f5efbce30 is 50, key is test_row_0/B:col10/1733145538674/Put/seqid=0 2024-12-02T13:19:01,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742255_1431 (size=12151) 2024-12-02T13:19:01,586 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/a4d8ccedc74b43c2a995fc9f5efbce30 2024-12-02T13:19:01,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ca8a62982c4540ff94f068c4099ff015 is 50, key is test_row_0/C:col10/1733145538674/Put/seqid=0 2024-12-02T13:19:01,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742256_1432 (size=12151) 2024-12-02T13:19:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:01,996 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ca8a62982c4540ff94f068c4099ff015 2024-12-02T13:19:01,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a885b0230d9a4e66b0695280657ddac2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2 2024-12-02T13:19:02,006 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2, entries=150, sequenceid=242, filesize=30.4 K 2024-12-02T13:19:02,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/a4d8ccedc74b43c2a995fc9f5efbce30 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30 2024-12-02T13:19:02,011 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T13:19:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ca8a62982c4540ff94f068c4099ff015 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015 2024-12-02T13:19:02,016 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015, entries=150, sequenceid=242, filesize=11.9 K 2024-12-02T13:19:02,016 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2fbac5b7f28671050128676c4e02d4f1 in 1267ms, sequenceid=242, compaction requested=true 2024-12-02T13:19:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:02,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-02T13:19:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-02T13:19:02,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-02T13:19:02,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-12-02T13:19:02,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.4260 sec 2024-12-02T13:19:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:02,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:02,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba61b1007b934c9aaf9284bc430dcd3c_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:02,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742257_1433 (size=14844) 2024-12-02T13:19:02,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145602137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145602137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145602254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145602254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:02,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ba61b1007b934c9aaf9284bc430dcd3c_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba61b1007b934c9aaf9284bc430dcd3c_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:02,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/42939404648e4df69d881b70cbc84180, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:02,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/42939404648e4df69d881b70cbc84180 is 175, key is test_row_0/A:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145602464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145602464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742258_1434 (size=39799) 2024-12-02T13:19:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T13:19:02,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:02,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:02,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-02T13:19:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:02,745 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:02,746 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:02,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:02,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145602784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145602784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:02,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:02,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/42939404648e4df69d881b70cbc84180 2024-12-02T13:19:02,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 is 50, key is test_row_0/B:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:02,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742259_1435 (size=12201) 2024-12-02T13:19:02,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:02,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:02,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:02,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:03,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:03,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 2024-12-02T13:19:03,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/142c8eaac1a54b718ca49b1db06422b3 is 50, key is test_row_0/C:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:03,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742260_1436 (size=12201) 2024-12-02T13:19:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145603294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:03,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145603294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:03,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:03,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:03,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:03,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/142c8eaac1a54b718ca49b1db06422b3 2024-12-02T13:19:03,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/42939404648e4df69d881b70cbc84180 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180 2024-12-02T13:19:03,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180, entries=200, sequenceid=257, filesize=38.9 K 2024-12-02T13:19:03,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 2024-12-02T13:19:03,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5, entries=150, sequenceid=257, filesize=11.9 K 2024-12-02T13:19:03,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/142c8eaac1a54b718ca49b1db06422b3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3 2024-12-02T13:19:03,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3, entries=150, sequenceid=257, filesize=11.9 K 2024-12-02T13:19:03,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2fbac5b7f28671050128676c4e02d4f1 in 1659ms, sequenceid=257, compaction requested=true 2024-12-02T13:19:03,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:03,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:03,709 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:03,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:03,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:03,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:03,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:03,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:03,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:03,710 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133626 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:03,710 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:03,710 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:19:03,710 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:19:03,710 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,710 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,710 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=130.5 K 2024-12-02T13:19:03,710 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/1c6d3af48b5d49bba5bb374dd2cffd12, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=48.0 K 2024-12-02T13:19:03,710 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,710 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180] 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6d3af48b5d49bba5bb374dd2cffd12, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536818 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffbd3ba1ed35431190a826585c301d40, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536818 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 220df5d32fdd409eb08481e97b16c69c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733145537484 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3445ae8db11d497e9160d732db727b43, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733145537484 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d8ccedc74b43c2a995fc9f5efbce30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733145538674 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a885b0230d9a4e66b0695280657ddac2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733145538674 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5cb7eaf38a48f8ab48c63d0c3a6eb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:03,711 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42939404648e4df69d881b70cbc84180, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:03,717 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:03,718 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:03,718 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/64a1e931dc6241b7b73943bca9919881 is 50, key is test_row_0/B:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:03,719 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202690cfc4a371d4f5794e1f64b5d005887_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:03,721 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202690cfc4a371d4f5794e1f64b5d005887_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:03,721 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202690cfc4a371d4f5794e1f64b5d005887_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:03,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742261_1437 (size=12849) 2024-12-02T13:19:03,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742262_1438 (size=4469) 2024-12-02T13:19:03,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-02T13:19:03,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:03,812 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:03,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:03,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a54ba3846b8e476985191ed9440b6c60_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145542136/Put/seqid=0 2024-12-02T13:19:03,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742263_1439 (size=12454) 2024-12-02T13:19:03,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:03,824 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a54ba3846b8e476985191ed9440b6c60_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a54ba3846b8e476985191ed9440b6c60_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af149ce004aa439bbf4db2a888195cd4, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af149ce004aa439bbf4db2a888195cd4 is 175, key is test_row_0/A:col10/1733145542136/Put/seqid=0 2024-12-02T13:19:03,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742264_1440 (size=31255) 2024-12-02T13:19:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:04,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:04,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39800 deadline: 1733145604064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145604067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145604067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,069 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,131 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/64a1e931dc6241b7b73943bca9919881 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/64a1e931dc6241b7b73943bca9919881 2024-12-02T13:19:04,134 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:04,135 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 64a1e931dc6241b7b73943bca9919881(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:04,135 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/25cdc77291944f6a85c81f66a25058d3 is 175, key is test_row_0/A:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:04,135 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:04,135 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=12, startTime=1733145543709; duration=0sec 2024-12-02T13:19:04,135 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:04,135 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:19:04,135 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:04,136 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:04,136 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:19:04,136 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:04,136 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/1e42f3391b714974931f10a62cd76189, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=48.0 K 2024-12-02T13:19:04,137 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e42f3391b714974931f10a62cd76189, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733145536818 2024-12-02T13:19:04,137 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a15b2b6dd890469b80c45c48f7906731, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733145537484 2024-12-02T13:19:04,137 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8a62982c4540ff94f068c4099ff015, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733145538674 2024-12-02T13:19:04,137 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 142c8eaac1a54b718ca49b1db06422b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:04,149 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:04,149 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/5f7a69ec3ac1425bb44dba0f693bdb04 is 50, key is test_row_0/C:col10/1733145542049/Put/seqid=0 2024-12-02T13:19:04,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742265_1441 (size=31803) 2024-12-02T13:19:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742266_1442 (size=12849) 2024-12-02T13:19:04,156 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/5f7a69ec3ac1425bb44dba0f693bdb04 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5f7a69ec3ac1425bb44dba0f693bdb04 2024-12-02T13:19:04,162 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into 5f7a69ec3ac1425bb44dba0f693bdb04(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:04,163 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:04,163 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=12, startTime=1733145543709; duration=0sec 2024-12-02T13:19:04,163 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:04,163 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:19:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145604174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145604174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,229 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af149ce004aa439bbf4db2a888195cd4 2024-12-02T13:19:04,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/4ce0ae91f7584fe38022d88fceff59aa is 50, key is test_row_0/B:col10/1733145542136/Put/seqid=0 2024-12-02T13:19:04,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742267_1443 (size=12301) 2024-12-02T13:19:04,244 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/4ce0ae91f7584fe38022d88fceff59aa 2024-12-02T13:19:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f90ae1ac19ad465a8dca976c0d024c42 is 50, key is test_row_0/C:col10/1733145542136/Put/seqid=0 2024-12-02T13:19:04,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742268_1444 (size=12301) 2024-12-02T13:19:04,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145604304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145604304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,312 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,312 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145604384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145604384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,554 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/25cdc77291944f6a85c81f66a25058d3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3 2024-12-02T13:19:04,557 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into 25cdc77291944f6a85c81f66a25058d3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:04,557 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:04,557 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=12, startTime=1733145543709; duration=0sec 2024-12-02T13:19:04,558 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:04,558 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:19:04,654 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f90ae1ac19ad465a8dca976c0d024c42 2024-12-02T13:19:04,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/af149ce004aa439bbf4db2a888195cd4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4 2024-12-02T13:19:04,661 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4, entries=150, sequenceid=274, filesize=30.5 K 2024-12-02T13:19:04,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/4ce0ae91f7584fe38022d88fceff59aa as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa 2024-12-02T13:19:04,664 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa, entries=150, sequenceid=274, filesize=12.0 K 2024-12-02T13:19:04,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f90ae1ac19ad465a8dca976c0d024c42 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42 2024-12-02T13:19:04,668 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42, entries=150, sequenceid=274, filesize=12.0 K 2024-12-02T13:19:04,668 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2fbac5b7f28671050128676c4e02d4f1 in 856ms, sequenceid=274, compaction requested=false 2024-12-02T13:19:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-02T13:19:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-02T13:19:04,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T13:19:04,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-12-02T13:19:04,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.9280 sec 2024-12-02T13:19:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:04,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:04,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:04,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120248be99d3be06483584e4a0d2450c658a_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:04,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742269_1445 (size=14994) 2024-12-02T13:19:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145604737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145604745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145604854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145604854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:04,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T13:19:04,894 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:04,895 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-02T13:19:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:04,896 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:04,897 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:04,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:05,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T13:19:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:05,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145605064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145605064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:05,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120248be99d3be06483584e4a0d2450c658a_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120248be99d3be06483584e4a0d2450c658a_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:05,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/e980610b6c9f4098957364b052af63ff, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/e980610b6c9f4098957364b052af63ff is 175, key is test_row_0/A:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742270_1446 (size=39949) 2024-12-02T13:19:05,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/e980610b6c9f4098957364b052af63ff 2024-12-02T13:19:05,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b6e3a2002eed4f78a635670965d9a4ad is 50, key is test_row_0/B:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742271_1447 (size=12301) 2024-12-02T13:19:05,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b6e3a2002eed4f78a635670965d9a4ad 2024-12-02T13:19:05,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c962b059847d46da811e227701e048de is 50, key is test_row_0/C:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742272_1448 (size=12301) 2024-12-02T13:19:05,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c962b059847d46da811e227701e048de 2024-12-02T13:19:05,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/e980610b6c9f4098957364b052af63ff as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff 2024-12-02T13:19:05,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff, entries=200, sequenceid=293, filesize=39.0 K 2024-12-02T13:19:05,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b6e3a2002eed4f78a635670965d9a4ad as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad 2024-12-02T13:19:05,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T13:19:05,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c962b059847d46da811e227701e048de as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de 2024-12-02T13:19:05,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de, entries=150, sequenceid=293, filesize=12.0 K 2024-12-02T13:19:05,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2fbac5b7f28671050128676c4e02d4f1 in 468ms, sequenceid=293, compaction requested=true 2024-12-02T13:19:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:05,165 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:05,165 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:05,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:19:05,166 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,166 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,166 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/64a1e931dc6241b7b73943bca9919881, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.6 K 2024-12-02T13:19:05,166 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=100.6 K 2024-12-02T13:19:05,166 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff] 2024-12-02T13:19:05,166 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a1e931dc6241b7b73943bca9919881, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:05,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25cdc77291944f6a85c81f66a25058d3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:05,167 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce0ae91f7584fe38022d88fceff59aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733145542100 2024-12-02T13:19:05,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting af149ce004aa439bbf4db2a888195cd4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733145542100 2024-12-02T13:19:05,167 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e3a2002eed4f78a635670965d9a4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:05,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e980610b6c9f4098957364b052af63ff, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:05,172 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,173 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:05,173 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/120293960f3f4a078ff263a919c1a0d8 is 50, key is test_row_0/B:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,175 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202378da2adc2ea4eaea987b3b01899b1fc_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,177 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202378da2adc2ea4eaea987b3b01899b1fc_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,177 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202378da2adc2ea4eaea987b3b01899b1fc_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742273_1449 (size=13051) 2024-12-02T13:19:05,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742274_1450 (size=4469) 2024-12-02T13:19:05,189 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#374 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:05,190 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0c8e4134ff534c94b491af77ba8c21bb is 175, key is test_row_0/A:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742275_1451 (size=32005) 2024-12-02T13:19:05,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,202 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:05,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:05,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028aa28479c90c4b22b8b9cd2043f86e65_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145544743/Put/seqid=0 2024-12-02T13:19:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:05,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742276_1452 (size=12454) 2024-12-02T13:19:05,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:05,221 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028aa28479c90c4b22b8b9cd2043f86e65_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa28479c90c4b22b8b9cd2043f86e65_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f6ec61d70c544a2ab8ca0e8d5444aac4, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:05,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f6ec61d70c544a2ab8ca0e8d5444aac4 is 175, key is test_row_0/A:col10/1733145544743/Put/seqid=0 2024-12-02T13:19:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742277_1453 (size=31255) 2024-12-02T13:19:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:05,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. as already flushing 2024-12-02T13:19:05,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145605415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145605422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:05,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145605533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145605534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,592 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/120293960f3f4a078ff263a919c1a0d8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/120293960f3f4a078ff263a919c1a0d8 2024-12-02T13:19:05,595 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 120293960f3f4a078ff263a919c1a0d8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:05,595 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:05,595 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145545165; duration=0sec 2024-12-02T13:19:05,596 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:05,596 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:19:05,596 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:05,596 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:05,596 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:19:05,596 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:05,596 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5f7a69ec3ac1425bb44dba0f693bdb04, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.6 K 2024-12-02T13:19:05,597 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f7a69ec3ac1425bb44dba0f693bdb04, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733145540875 2024-12-02T13:19:05,597 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f90ae1ac19ad465a8dca976c0d024c42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733145542100 2024-12-02T13:19:05,598 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c962b059847d46da811e227701e048de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:05,598 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/0c8e4134ff534c94b491af77ba8c21bb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb 2024-12-02T13:19:05,602 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into 0c8e4134ff534c94b491af77ba8c21bb(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:05,602 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:05,602 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145545165; duration=0sec 2024-12-02T13:19:05,602 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:05,602 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:19:05,604 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:05,604 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c15916b8fd0641edbbaa4018c1fe70c2 is 50, key is test_row_0/C:col10/1733145544066/Put/seqid=0 2024-12-02T13:19:05,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742278_1454 (size=13051) 2024-12-02T13:19:05,610 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/c15916b8fd0641edbbaa4018c1fe70c2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c15916b8fd0641edbbaa4018c1fe70c2 2024-12-02T13:19:05,614 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into c15916b8fd0641edbbaa4018c1fe70c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:05,614 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:05,614 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145545166; duration=0sec 2024-12-02T13:19:05,614 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:05,614 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:19:05,627 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f6ec61d70c544a2ab8ca0e8d5444aac4 2024-12-02T13:19:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b62933a9daaa43a4959d25a91e4cee2c is 50, key is test_row_0/B:col10/1733145544743/Put/seqid=0 2024-12-02T13:19:05,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742279_1455 (size=12301) 2024-12-02T13:19:05,664 INFO [Thread-1642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1642. 2024-12-02T13:19:05,664 DEBUG [Thread-1642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:05,664 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,664 DEBUG [Thread-1642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:05,665 INFO [Thread-1640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1640. 2024-12-02T13:19:05,665 DEBUG [Thread-1640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:05,665 DEBUG [Thread-1640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,665 DEBUG [Thread-1640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:05,666 INFO [Thread-1646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1646. 2024-12-02T13:19:05,666 DEBUG [Thread-1646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:05,666 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,666 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:05,672 INFO [Thread-1644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1644. 2024-12-02T13:19:05,672 INFO [Thread-1648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1648. 2024-12-02T13:19:05,672 DEBUG [Thread-1644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:05,672 DEBUG [Thread-1648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:05,672 DEBUG [Thread-1644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,672 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,672 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,672 DEBUG [Thread-1644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:05,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:05,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:05,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145605744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145605744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:05,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:06,038 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b62933a9daaa43a4959d25a91e4cee2c 2024-12-02T13:19:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/27764cf995d146d9a590a610f6df84d6 is 50, key is test_row_0/C:col10/1733145544743/Put/seqid=0 2024-12-02T13:19:06,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742280_1456 (size=12301) 2024-12-02T13:19:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39840 deadline: 1733145606054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39790 deadline: 1733145606054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:06,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:06,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39852 deadline: 1733145606325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39834 deadline: 1733145606325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:06,326 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:06,326 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 2fbac5b7f28671050128676c4e02d4f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:06,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:06,454 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/27764cf995d146d9a590a610f6df84d6 2024-12-02T13:19:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f6ec61d70c544a2ab8ca0e8d5444aac4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4 2024-12-02T13:19:06,469 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4, entries=150, sequenceid=309, filesize=30.5 K 2024-12-02T13:19:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/b62933a9daaa43a4959d25a91e4cee2c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c 2024-12-02T13:19:06,472 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T13:19:06,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/27764cf995d146d9a590a610f6df84d6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6 2024-12-02T13:19:06,475 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6, entries=150, sequenceid=309, filesize=12.0 K 2024-12-02T13:19:06,476 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2fbac5b7f28671050128676c4e02d4f1 in 1274ms, sequenceid=309, compaction requested=false 2024-12-02T13:19:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T13:19:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T13:19:06,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T13:19:06,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-12-02T13:19:06,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5840 sec 2024-12-02T13:19:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:06,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T13:19:06,566 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-12-02T13:19:06,566 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-12-02T13:19:06,566 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:06,566 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:06,566 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:06,566 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:06,566 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:06,566 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:06,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:06,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:06,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027ffee9eb7a794b4b89f22d35d5dafd11_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_0/A:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:06,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742281_1457 (size=12454) 2024-12-02T13:19:06,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:19:06,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:06,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027ffee9eb7a794b4b89f22d35d5dafd11_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ffee9eb7a794b4b89f22d35d5dafd11_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:06,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f12e9aeeeb324500a009240c94905c4b, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:06,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f12e9aeeeb324500a009240c94905c4b is 175, key is test_row_0/A:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:06,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742282_1458 (size=31255) 2024-12-02T13:19:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T13:19:07,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:07,396 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f12e9aeeeb324500a009240c94905c4b 2024-12-02T13:19:07,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/f1054e4ee6a940c1b8683540da2e31ba is 50, key is test_row_0/B:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:07,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742283_1459 (size=12301) 2024-12-02T13:19:07,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/f1054e4ee6a940c1b8683540da2e31ba 2024-12-02T13:19:07,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ecaefef1fa784f0cb7563dffd122528e is 50, key is test_row_0/C:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:07,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742284_1460 (size=12301) 2024-12-02T13:19:08,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ecaefef1fa784f0cb7563dffd122528e 2024-12-02T13:19:08,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/f12e9aeeeb324500a009240c94905c4b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b 2024-12-02T13:19:08,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b, entries=150, sequenceid=330, filesize=30.5 K 2024-12-02T13:19:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/f1054e4ee6a940c1b8683540da2e31ba as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba 2024-12-02T13:19:08,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba, entries=150, sequenceid=330, filesize=12.0 K 2024-12-02T13:19:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/ecaefef1fa784f0cb7563dffd122528e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e 2024-12-02T13:19:08,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e, entries=150, sequenceid=330, filesize=12.0 K 2024-12-02T13:19:08,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 2fbac5b7f28671050128676c4e02d4f1 in 1686ms, sequenceid=330, compaction requested=true 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:08,252 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:08,252 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2fbac5b7f28671050128676c4e02d4f1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/A is initiating minor compaction (all files) 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/B is initiating minor compaction (all files) 2024-12-02T13:19:08,253 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/B in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:08,253 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/A in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:08,253 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=92.3 K 2024-12-02T13:19:08,253 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/120293960f3f4a078ff263a919c1a0d8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.8 K 2024-12-02T13:19:08,253 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b] 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 120293960f3f4a078ff263a919c1a0d8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:08,253 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8e4134ff534c94b491af77ba8c21bb, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:08,254 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b62933a9daaa43a4959d25a91e4cee2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733145544720 2024-12-02T13:19:08,254 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ec61d70c544a2ab8ca0e8d5444aac4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733145544720 2024-12-02T13:19:08,254 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f1054e4ee6a940c1b8683540da2e31ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733145545414 2024-12-02T13:19:08,254 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12e9aeeeb324500a009240c94905c4b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733145545414 2024-12-02T13:19:08,262 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#B#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:08,263 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/9a6807bc60ea4081978b92365b1c967a is 50, key is test_row_0/B:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:08,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742285_1461 (size=13153) 2024-12-02T13:19:08,266 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:08,268 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ed04c9397ee143578dd2df35b5007298_2fbac5b7f28671050128676c4e02d4f1 store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:08,283 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ed04c9397ee143578dd2df35b5007298_2fbac5b7f28671050128676c4e02d4f1, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:08,284 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ed04c9397ee143578dd2df35b5007298_2fbac5b7f28671050128676c4e02d4f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742286_1462 (size=4469) 2024-12-02T13:19:08,673 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/9a6807bc60ea4081978b92365b1c967a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/9a6807bc60ea4081978b92365b1c967a 2024-12-02T13:19:08,679 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/B of 2fbac5b7f28671050128676c4e02d4f1 into 9a6807bc60ea4081978b92365b1c967a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:08,679 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:08,679 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/B, priority=13, startTime=1733145548252; duration=0sec 2024-12-02T13:19:08,680 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:08,680 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:B 2024-12-02T13:19:08,680 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:08,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:08,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 2fbac5b7f28671050128676c4e02d4f1/C is initiating minor compaction (all files) 2024-12-02T13:19:08,681 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2fbac5b7f28671050128676c4e02d4f1/C in TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:08,681 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c15916b8fd0641edbbaa4018c1fe70c2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp, totalSize=36.8 K 2024-12-02T13:19:08,681 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c15916b8fd0641edbbaa4018c1fe70c2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733145544046 2024-12-02T13:19:08,682 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 27764cf995d146d9a590a610f6df84d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733145544720 2024-12-02T13:19:08,682 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ecaefef1fa784f0cb7563dffd122528e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733145545414 2024-12-02T13:19:08,688 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#A#compaction#384 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:08,688 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a5e4ea2722bb4840a41f6dfe904b08cc is 175, key is test_row_0/A:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:08,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742287_1463 (size=32107) 2024-12-02T13:19:08,692 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2fbac5b7f28671050128676c4e02d4f1#C#compaction#385 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:08,692 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/6df91a2c48f74754a88f976b5703a62e is 50, key is test_row_0/C:col10/1733145546564/Put/seqid=0 2024-12-02T13:19:08,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742288_1464 (size=13153) 2024-12-02T13:19:09,103 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/a5e4ea2722bb4840a41f6dfe904b08cc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a5e4ea2722bb4840a41f6dfe904b08cc 2024-12-02T13:19:09,105 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/6df91a2c48f74754a88f976b5703a62e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/6df91a2c48f74754a88f976b5703a62e 2024-12-02T13:19:09,109 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/A of 2fbac5b7f28671050128676c4e02d4f1 into a5e4ea2722bb4840a41f6dfe904b08cc(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:09,109 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:09,109 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/A, priority=13, startTime=1733145548252; duration=0sec 2024-12-02T13:19:09,109 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:09,109 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:A 2024-12-02T13:19:09,110 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2fbac5b7f28671050128676c4e02d4f1/C of 2fbac5b7f28671050128676c4e02d4f1 into 6df91a2c48f74754a88f976b5703a62e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:09,110 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2fbac5b7f28671050128676c4e02d4f1: 2024-12-02T13:19:09,110 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., storeName=2fbac5b7f28671050128676c4e02d4f1/C, priority=13, startTime=1733145548252; duration=0sec 2024-12-02T13:19:09,110 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:09,110 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2fbac5b7f28671050128676c4e02d4f1:C 2024-12-02T13:19:10,339 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-12-02T13:19:10,340 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:10,340 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:10,340 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:10,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:10,375 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-12-02T13:19:10,375 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:10,376 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:10,376 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:10,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:10,968 WARN [regionserver/06e3e18f2515:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-02T13:19:10,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b] to archive 2024-12-02T13:19:10,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:10,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/d554de96a5dd4a0d90cda44aeecac84c 2024-12-02T13:19:10,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0bbbf0defb9843e7b30387632646eaf1 2024-12-02T13:19:10,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af56c8412f8e42759485307f20cdd43f 2024-12-02T13:19:10,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/b602d9da2a554a5e8c81753991600e63 2024-12-02T13:19:10,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/019bedd0fd7e4389bb7a2b692cc9d75b 2024-12-02T13:19:10,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/c49454930fcf4311a370619881bd9111 2024-12-02T13:19:10,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a80025741f7748aa835e25da62442f98 2024-12-02T13:19:10,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4b2ca8d68ad14f35a4bd8d9349c5f588 2024-12-02T13:19:10,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4c3e85cd0c6a4cf8b1ceb220c7776ed9 2024-12-02T13:19:10,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/4a5b1c5f1f6142479e02e2e1a1fa2e62 2024-12-02T13:19:10,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8ead1ea3fb464583948ad761045fd792 2024-12-02T13:19:10,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/997be22454ad4a76b7b5cc4d3a65fec3 2024-12-02T13:19:10,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0acb8a25500a4ca8b45b628540c72555 2024-12-02T13:19:10,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/caecbe24e4844b89a88dd4110570e885 2024-12-02T13:19:10,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/8b76d8dd75314a19a4006a97a7c3b0dd 2024-12-02T13:19:10,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ffbd3ba1ed35431190a826585c301d40 2024-12-02T13:19:10,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/3445ae8db11d497e9160d732db727b43 2024-12-02T13:19:10,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a885b0230d9a4e66b0695280657ddac2 2024-12-02T13:19:10,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/42939404648e4df69d881b70cbc84180 2024-12-02T13:19:10,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/25cdc77291944f6a85c81f66a25058d3 2024-12-02T13:19:10,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/af149ce004aa439bbf4db2a888195cd4 2024-12-02T13:19:10,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/e980610b6c9f4098957364b052af63ff 2024-12-02T13:19:10,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/0c8e4134ff534c94b491af77ba8c21bb 2024-12-02T13:19:11,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f6ec61d70c544a2ab8ca0e8d5444aac4 2024-12-02T13:19:11,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/f12e9aeeeb324500a009240c94905c4b 2024-12-02T13:19:11,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/15440fd98b4a43bda0dfd36a3451634e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/fae691c7647c459f954edd32b2bbbc13, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b2f5d28b66dd4637b1c764e14ee4bce3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/66197c65f02e4045b2733cdd5273d426, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/1c6d3af48b5d49bba5bb374dd2cffd12, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/64a1e931dc6241b7b73943bca9919881, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/120293960f3f4a078ff263a919c1a0d8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba] to archive 2024-12-02T13:19:11,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:11,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2b096612b27e4f29a6046faba444e63e 2024-12-02T13:19:11,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/bf66718b0d124d0e9a8bf6ae0d7b2e09 2024-12-02T13:19:11,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/15440fd98b4a43bda0dfd36a3451634e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/15440fd98b4a43bda0dfd36a3451634e 2024-12-02T13:19:11,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/2f7132522cff4ff598058e76c123377d 2024-12-02T13:19:11,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e850005b4f9b4ced9c55f7b7db1f24d6 2024-12-02T13:19:11,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/fae691c7647c459f954edd32b2bbbc13 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/fae691c7647c459f954edd32b2bbbc13 2024-12-02T13:19:11,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e3f7e363d0a54ebc92ef478f3d829118 2024-12-02T13:19:11,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/e1e1bf2e974c4cf5a8268d5b877205e0 2024-12-02T13:19:11,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b2f5d28b66dd4637b1c764e14ee4bce3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b2f5d28b66dd4637b1c764e14ee4bce3 2024-12-02T13:19:11,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/732e68207dfa4f54a73a4a4a9c69e0d3 2024-12-02T13:19:11,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/ca1d10a95a9549448435689fbb6d4170 2024-12-02T13:19:11,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/66197c65f02e4045b2733cdd5273d426 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/66197c65f02e4045b2733cdd5273d426 2024-12-02T13:19:11,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/983e8d9186d141e6a207b5645a073dbf 2024-12-02T13:19:11,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b789afdf9cee42deb654d91cecfd82e6 2024-12-02T13:19:11,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/1c6d3af48b5d49bba5bb374dd2cffd12 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/1c6d3af48b5d49bba5bb374dd2cffd12 2024-12-02T13:19:11,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/6af6620be6434056b77eb147cbed89ad 2024-12-02T13:19:11,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/220df5d32fdd409eb08481e97b16c69c 2024-12-02T13:19:11,028 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/a4d8ccedc74b43c2a995fc9f5efbce30 2024-12-02T13:19:11,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/64a1e931dc6241b7b73943bca9919881 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/64a1e931dc6241b7b73943bca9919881 2024-12-02T13:19:11,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/8d5cb7eaf38a48f8ab48c63d0c3a6eb5 2024-12-02T13:19:11,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/4ce0ae91f7584fe38022d88fceff59aa 2024-12-02T13:19:11,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/120293960f3f4a078ff263a919c1a0d8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/120293960f3f4a078ff263a919c1a0d8 2024-12-02T13:19:11,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b6e3a2002eed4f78a635670965d9a4ad 2024-12-02T13:19:11,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/b62933a9daaa43a4959d25a91e4cee2c 2024-12-02T13:19:11,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/f1054e4ee6a940c1b8683540da2e31ba 2024-12-02T13:19:11,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/de1fd5de99ea4a32a9522d7712906cb7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/625f1d0068ea4bc99cc4bc96fc7e63f1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/95d0564be1f94d509c71cb5ba3152765, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e6c936adbd7a4063aaf826d5fab218b1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/1e42f3391b714974931f10a62cd76189, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5f7a69ec3ac1425bb44dba0f693bdb04, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c15916b8fd0641edbbaa4018c1fe70c2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e] to archive 2024-12-02T13:19:11,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:11,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a9893af2d24e4a42a9a4c2449e22433d 2024-12-02T13:19:11,046 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/b3ac63a39cd64ad996ed706e43621d95 2024-12-02T13:19:11,047 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/de1fd5de99ea4a32a9522d7712906cb7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/de1fd5de99ea4a32a9522d7712906cb7 2024-12-02T13:19:11,048 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c2a177bc19304dbbabc063d2df608a6c 2024-12-02T13:19:11,048 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/635cc4ba8ee146cf994a8b53dce19943 2024-12-02T13:19:11,049 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/625f1d0068ea4bc99cc4bc96fc7e63f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/625f1d0068ea4bc99cc4bc96fc7e63f1 2024-12-02T13:19:11,050 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/520d1d5c42bc42728ce4686e5125403d 2024-12-02T13:19:11,051 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/67cf127e246f4142abe98df31db6c4ae 2024-12-02T13:19:11,052 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/95d0564be1f94d509c71cb5ba3152765 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/95d0564be1f94d509c71cb5ba3152765 2024-12-02T13:19:11,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e188bf26862145d39d15d97b37531832 2024-12-02T13:19:11,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/314df6b2f10b4666b965312185083974 2024-12-02T13:19:11,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e6c936adbd7a4063aaf826d5fab218b1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/e6c936adbd7a4063aaf826d5fab218b1 2024-12-02T13:19:11,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/2b68d22299d749618718558c4bc2d7f2 2024-12-02T13:19:11,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/29a77b5219634dfcb34798a06361055d 2024-12-02T13:19:11,057 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/1e42f3391b714974931f10a62cd76189 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/1e42f3391b714974931f10a62cd76189 2024-12-02T13:19:11,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5622d0c18da849749d94fe468376643b 2024-12-02T13:19:11,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/a15b2b6dd890469b80c45c48f7906731 2024-12-02T13:19:11,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ca8a62982c4540ff94f068c4099ff015 2024-12-02T13:19:11,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5f7a69ec3ac1425bb44dba0f693bdb04 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/5f7a69ec3ac1425bb44dba0f693bdb04 2024-12-02T13:19:11,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/142c8eaac1a54b718ca49b1db06422b3 2024-12-02T13:19:11,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f90ae1ac19ad465a8dca976c0d024c42 2024-12-02T13:19:11,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c15916b8fd0641edbbaa4018c1fe70c2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c15916b8fd0641edbbaa4018c1fe70c2 2024-12-02T13:19:11,069 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/c962b059847d46da811e227701e048de 2024-12-02T13:19:11,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/27764cf995d146d9a590a610f6df84d6 2024-12-02T13:19:11,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/06e3e18f2515:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/ecaefef1fa784f0cb7563dffd122528e 2024-12-02T13:19:14,119 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-12-02T13:19:14,120 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:14,120 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:14,120 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:14,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4346 rows 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-12-02T13:19:14,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4342 rows 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4345 rows 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1449 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4346 rows 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1436 2024-12-02T13:19:14,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4306 rows 2024-12-02T13:19:14,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:19:14,122 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:19:14,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:14,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:14,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:14,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:19:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:14,128 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145554127"}]},"ts":"1733145554127"} 2024-12-02T13:19:14,129 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:19:14,129 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:19:14,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:19:14,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, UNASSIGN}] 2024-12-02T13:19:14,133 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, UNASSIGN 2024-12-02T13:19:14,133 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:14,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, UNASSIGN because future has completed 2024-12-02T13:19:14,135 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:19:14,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:14,288 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:14,288 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:19:14,289 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 2fbac5b7f28671050128676c4e02d4f1, disabling compactions & flushes 2024-12-02T13:19:14,289 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:14,289 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:14,289 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. after waiting 0 ms 2024-12-02T13:19:14,289 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:14,289 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 2fbac5b7f28671050128676c4e02d4f1 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=A 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=B 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2fbac5b7f28671050128676c4e02d4f1, store=C 2024-12-02T13:19:14,290 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:14,302 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b270c4b1997847fe9d4acea38853e0cb_2fbac5b7f28671050128676c4e02d4f1 is 50, key is test_row_2/A:col10/1733145554116/Put/seqid=0 2024-12-02T13:19:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742289_1465 (size=7374) 2024-12-02T13:19:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:14,707 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:14,716 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b270c4b1997847fe9d4acea38853e0cb_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b270c4b1997847fe9d4acea38853e0cb_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:14,717 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ed40886ef43449efb8668730de0ff8c7, store: [table=TestAcidGuarantees family=A region=2fbac5b7f28671050128676c4e02d4f1] 2024-12-02T13:19:14,718 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ed40886ef43449efb8668730de0ff8c7 is 175, key is test_row_2/A:col10/1733145554116/Put/seqid=0 2024-12-02T13:19:14,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742290_1466 (size=13865) 2024-12-02T13:19:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:15,123 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ed40886ef43449efb8668730de0ff8c7 2024-12-02T13:19:15,136 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/49c6c7d4393347cca48c0a4f62ded65c is 50, key is test_row_2/B:col10/1733145554116/Put/seqid=0 2024-12-02T13:19:15,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742291_1467 (size=7415) 2024-12-02T13:19:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:15,541 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/49c6c7d4393347cca48c0a4f62ded65c 2024-12-02T13:19:15,555 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f6a7f2143a7d43de9b77ef869e7c7a58 is 50, key is test_row_2/C:col10/1733145554116/Put/seqid=0 2024-12-02T13:19:15,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742292_1468 (size=7415) 2024-12-02T13:19:15,960 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f6a7f2143a7d43de9b77ef869e7c7a58 2024-12-02T13:19:15,970 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/A/ed40886ef43449efb8668730de0ff8c7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ed40886ef43449efb8668730de0ff8c7 2024-12-02T13:19:15,975 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ed40886ef43449efb8668730de0ff8c7, entries=50, sequenceid=339, filesize=13.5 K 2024-12-02T13:19:15,976 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/B/49c6c7d4393347cca48c0a4f62ded65c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/49c6c7d4393347cca48c0a4f62ded65c 2024-12-02T13:19:15,979 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/49c6c7d4393347cca48c0a4f62ded65c, entries=50, sequenceid=339, filesize=7.2 K 2024-12-02T13:19:15,979 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/.tmp/C/f6a7f2143a7d43de9b77ef869e7c7a58 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f6a7f2143a7d43de9b77ef869e7c7a58 2024-12-02T13:19:15,982 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f6a7f2143a7d43de9b77ef869e7c7a58, entries=50, sequenceid=339, filesize=7.2 K 2024-12-02T13:19:15,983 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2fbac5b7f28671050128676c4e02d4f1 in 1694ms, sequenceid=339, compaction requested=false 2024-12-02T13:19:15,987 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=4 2024-12-02T13:19:15,987 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. 2024-12-02T13:19:15,987 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 2fbac5b7f28671050128676c4e02d4f1: Waiting for close lock at 1733145554289Running coprocessor pre-close hooks at 1733145554289Disabling compacts and flushes for region at 1733145554289Disabling writes for close at 1733145554289Obtaining lock to block concurrent updates at 1733145554289Preparing flush snapshotting stores in 2fbac5b7f28671050128676c4e02d4f1 at 1733145554289Finished memstore snapshotting TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733145554290 (+1 ms)Flushing stores of TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1. at 1733145554292 (+2 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/A: creating writer at 1733145554292Flushing mob file 2fbac5b7f28671050128676c4e02d4f1/A: appending metadata at 1733145554302 (+10 ms)Flushing mob file 2fbac5b7f28671050128676c4e02d4f1/A: closing flushed file at 1733145554302Flushing 2fbac5b7f28671050128676c4e02d4f1/A: appending metadata at 1733145554717 (+415 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/A: closing flushed file at 1733145554717Flushing 2fbac5b7f28671050128676c4e02d4f1/B: creating writer at 1733145555132 (+415 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/B: appending metadata at 1733145555135 (+3 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/B: closing flushed file at 1733145555135Flushing 2fbac5b7f28671050128676c4e02d4f1/C: creating writer at 1733145555550 (+415 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/C: appending metadata at 1733145555554 (+4 ms)Flushing 2fbac5b7f28671050128676c4e02d4f1/C: closing flushed file at 1733145555554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55a21ff4: reopening flushed file at 1733145555969 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3199f916: reopening flushed file at 1733145555975 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73fe7a1: reopening flushed file at 1733145555979 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 2fbac5b7f28671050128676c4e02d4f1 in 1694ms, sequenceid=339, compaction requested=false at 1733145555983 (+4 ms)Writing region close event to WAL at 1733145555984 (+1 ms)Running coprocessor post-close hooks at 1733145555987 (+3 ms)Closed at 1733145555987 2024-12-02T13:19:15,989 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:15,989 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2fbac5b7f28671050128676c4e02d4f1, regionState=CLOSED 2024-12-02T13:19:15,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:15,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T13:19:15,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 2fbac5b7f28671050128676c4e02d4f1, server=06e3e18f2515,45693,1733145428280 in 1.8560 sec 2024-12-02T13:19:15,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-12-02T13:19:15,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2fbac5b7f28671050128676c4e02d4f1, UNASSIGN in 1.8610 sec 2024-12-02T13:19:15,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-02T13:19:15,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8650 sec 2024-12-02T13:19:15,997 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145555997"}]},"ts":"1733145555997"} 2024-12-02T13:19:15,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:19:15,998 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:19:15,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8740 sec 2024-12-02T13:19:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T13:19:16,284 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:16,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:19:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,287 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T13:19:16,288 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,293 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits] 2024-12-02T13:19:16,297 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a5e4ea2722bb4840a41f6dfe904b08cc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/a5e4ea2722bb4840a41f6dfe904b08cc 2024-12-02T13:19:16,298 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ed40886ef43449efb8668730de0ff8c7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/A/ed40886ef43449efb8668730de0ff8c7 2024-12-02T13:19:16,300 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/49c6c7d4393347cca48c0a4f62ded65c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/49c6c7d4393347cca48c0a4f62ded65c 2024-12-02T13:19:16,301 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/9a6807bc60ea4081978b92365b1c967a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/B/9a6807bc60ea4081978b92365b1c967a 2024-12-02T13:19:16,303 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/6df91a2c48f74754a88f976b5703a62e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/6df91a2c48f74754a88f976b5703a62e 2024-12-02T13:19:16,304 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f6a7f2143a7d43de9b77ef869e7c7a58 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/C/f6a7f2143a7d43de9b77ef869e7c7a58 2024-12-02T13:19:16,306 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits/342.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1/recovered.edits/342.seqid 2024-12-02T13:19:16,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,307 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:19:16,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:19:16,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T13:19:16,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020743b5e33c08459693c1f9eca534c1ae_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020743b5e33c08459693c1f9eca534c1ae_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020b28dfb6bab14cd293252280d97cc4d9_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020b28dfb6bab14cd293252280d97cc4d9_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022bc8bbceeb6143ffb4b8f8d5fcd1b296_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022bc8bbceeb6143ffb4b8f8d5fcd1b296_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,314 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd0fe4643b54c1fb7568f3d18ef4445_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023cd0fe4643b54c1fb7568f3d18ef4445_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,315 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120248be99d3be06483584e4a0d2450c658a_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120248be99d3be06483584e4a0d2450c658a_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,316 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ad7e1f6205243778f94cd20cb1d0554_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ad7e1f6205243778f94cd20cb1d0554_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,317 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120267cfdd7c5cc6476bb10237bfab408743_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120267cfdd7c5cc6476bb10237bfab408743_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,318 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ace78c6a855484087d608ea2e199fcb_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026ace78c6a855484087d608ea2e199fcb_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,319 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ffee9eb7a794b4b89f22d35d5dafd11_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412027ffee9eb7a794b4b89f22d35d5dafd11_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,320 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120283e167add59b4ceebf0bb3a3c8b926ea_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120283e167add59b4ceebf0bb3a3c8b926ea_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,321 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa28479c90c4b22b8b9cd2043f86e65_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028aa28479c90c4b22b8b9cd2043f86e65_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,322 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ca3175e7db74909ae9fb41c56a75250_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ca3175e7db74909ae9fb41c56a75250_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,323 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029f5ed0b208074ccf8e97725da2897081_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029f5ed0b208074ccf8e97725da2897081_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,324 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a54ba3846b8e476985191ed9440b6c60_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a54ba3846b8e476985191ed9440b6c60_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,325 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202abc70673b179406cb1e03a192b80832e_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202abc70673b179406cb1e03a192b80832e_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,325 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b270c4b1997847fe9d4acea38853e0cb_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b270c4b1997847fe9d4acea38853e0cb_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,326 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba61b1007b934c9aaf9284bc430dcd3c_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202ba61b1007b934c9aaf9284bc430dcd3c_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,327 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eebb31580c1740f1a9cb72be89f31e29_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eebb31580c1740f1a9cb72be89f31e29_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,328 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f221a8409fd84a4b9e28563f845bb5f4_2fbac5b7f28671050128676c4e02d4f1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f221a8409fd84a4b9e28563f845bb5f4_2fbac5b7f28671050128676c4e02d4f1 2024-12-02T13:19:16,328 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:19:16,330 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,333 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:19:16,335 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:19:16,336 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,336 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:19:16,336 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145556336"}]},"ts":"9223372036854775807"} 2024-12-02T13:19:16,338 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:19:16,338 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2fbac5b7f28671050128676c4e02d4f1, NAME => 'TestAcidGuarantees,,1733145522495.2fbac5b7f28671050128676c4e02d4f1.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:19:16,338 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:19:16,338 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145556338"}]},"ts":"9223372036854775807"} 2024-12-02T13:19:16,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:19:16,340 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-12-02T13:19:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T13:19:16,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:19:16,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:16,403 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 252), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 499), ProcessCount=11 (was 11), AvailableMemoryMB=2531 (was 2579) 2024-12-02T13:19:16,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=2530 2024-12-02T13:19:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:19:16,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:19:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:16,413 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:19:16,413 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:16,413 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-12-02T13:19:16,414 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:19:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742293_1469 (size=1061) 2024-12-02T13:19:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:16,820 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:19:16,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742294_1470 (size=53) 2024-12-02T13:19:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:17,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:19:17,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a6dac54182c9e8f0ab33f471f7aac1b1, disabling compactions & flushes 2024-12-02T13:19:17,228 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. after waiting 0 ms 2024-12-02T13:19:17,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,228 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6dac54182c9e8f0ab33f471f7aac1b1: Waiting for close lock at 1733145557227Disabling compacts and flushes for region at 1733145557227Disabling writes for close at 1733145557228 (+1 ms)Writing region close event to WAL at 1733145557228Closed at 1733145557228 2024-12-02T13:19:17,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:19:17,232 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145557231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145557231"}]},"ts":"1733145557231"} 2024-12-02T13:19:17,236 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:19:17,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:19:17,239 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145557238"}]},"ts":"1733145557238"} 2024-12-02T13:19:17,240 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:19:17,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, ASSIGN}] 2024-12-02T13:19:17,242 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, ASSIGN 2024-12-02T13:19:17,242 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:19:17,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=a6dac54182c9e8f0ab33f471f7aac1b1, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:17,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, ASSIGN because future has completed 2024-12-02T13:19:17,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:17,552 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,552 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:19:17,553 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,553 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:19:17,553 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,553 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:17,554 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,555 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:17,556 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6dac54182c9e8f0ab33f471f7aac1b1 columnFamilyName A 2024-12-02T13:19:17,556 DEBUG [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:17,556 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(327): Store=a6dac54182c9e8f0ab33f471f7aac1b1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:17,556 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,557 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:17,558 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6dac54182c9e8f0ab33f471f7aac1b1 columnFamilyName B 2024-12-02T13:19:17,558 DEBUG [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:17,558 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(327): Store=a6dac54182c9e8f0ab33f471f7aac1b1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:17,558 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,561 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:17,561 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6dac54182c9e8f0ab33f471f7aac1b1 columnFamilyName C 2024-12-02T13:19:17,561 DEBUG [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:17,562 INFO [StoreOpener-a6dac54182c9e8f0ab33f471f7aac1b1-1 {}] regionserver.HStore(327): Store=a6dac54182c9e8f0ab33f471f7aac1b1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:17,562 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,562 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,562 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,563 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,563 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,563 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,564 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:19:17,565 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,566 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:19:17,567 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened a6dac54182c9e8f0ab33f471f7aac1b1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59383946, jitterRate=-0.11511024832725525}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:19:17,567 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:17,568 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for a6dac54182c9e8f0ab33f471f7aac1b1: Running coprocessor pre-open hook at 1733145557553Writing region info on filesystem at 1733145557553Initializing all the Stores at 1733145557554 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145557554Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145557554Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145557554Cleaning up temporary data from old regions at 1733145557564 (+10 ms)Running coprocessor post-open hooks at 1733145557567 (+3 ms)Region opened successfully at 1733145557568 (+1 ms) 2024-12-02T13:19:17,568 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., pid=130, masterSystemTime=1733145557549 2024-12-02T13:19:17,570 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,570 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:17,571 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=a6dac54182c9e8f0ab33f471f7aac1b1, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:17,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:17,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T13:19:17,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 in 177 msec 2024-12-02T13:19:17,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-02T13:19:17,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, ASSIGN in 335 msec 2024-12-02T13:19:17,578 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:19:17,579 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145557578"}]},"ts":"1733145557578"} 2024-12-02T13:19:17,580 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:19:17,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:19:17,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-12-02T13:19:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T13:19:18,565 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:18,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,568 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d79d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,570 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36711780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:19:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:19:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@621c8be5 2024-12-02T13:19:18,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:19:18,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:19:18,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,577 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de478a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,578 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d24bac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,580 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15be9baa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3036be10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49090d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80cfa14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622fd01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,589 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd25f2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c368405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f62810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,595 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e44b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,597 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0ea174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,599 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3860393d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,600 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ba649f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,602 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@249b576c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d327ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,605 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24af912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358e6b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:18,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:18,609 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8195f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:18,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39712055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:18,611 DEBUG [Thread-2110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,613 DEBUG [Thread-2114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,613 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,615 DEBUG [Thread-2118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,615 DEBUG [Thread-2099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:18,616 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,617 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,618 DEBUG [Thread-2107 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,618 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T13:19:18,619 DEBUG [Thread-2107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,620 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,620 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:18,620 DEBUG [Thread-2112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,621 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:18,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:18,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2] 2024-12-02T13:19:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:18,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:18,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2eb34dd8fc034d09924e3b756ddc92e3 is 50, key is test_row_0/A:col10/1733145558630/Put/seqid=0 2024-12-02T13:19:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742295_1471 (size=12001) 2024-12-02T13:19:18,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145558649, max timestamp=1733145558677Num uniques 859; 2024-12-02T13:19:18,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:19:18,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2eb34dd8fc034d09924e3b756ddc92e3 2024-12-02T13:19:18,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145558649, max timestamp=1733145558677Num uniques 859; 2024-12-02T13:19:18,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733145558649, max timestamp=1733145558677Num uniques 859; 2024-12-02T13:19:18,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:19:18,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T13:19:18,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/83a8690292af433fb1f0d2fe15a17244 is 50, key is test_row_0/B:col10/1733145558630/Put/seqid=0 2024-12-02T13:19:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145618713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145618715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145618728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145618730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145618730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742296_1472 (size=12001) 2024-12-02T13:19:18,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:18,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:18,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145618834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145618834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145618844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145618854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145618854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:18,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:18,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:18,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:18,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:18,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145619044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145619044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145619064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145619073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145619073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/83a8690292af433fb1f0d2fe15a17244 2024-12-02T13:19:19,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e2885e920c8246a9a28fac9864ff61fd is 50, key is test_row_0/C:col10/1733145558630/Put/seqid=0 2024-12-02T13:19:19,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742297_1473 (size=12001) 2024-12-02T13:19:19,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:19,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145619354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145619368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145619373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145619384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145619384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:19,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e2885e920c8246a9a28fac9864ff61fd 2024-12-02T13:19:19,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2eb34dd8fc034d09924e3b756ddc92e3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3 2024-12-02T13:19:19,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T13:19:19,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/83a8690292af433fb1f0d2fe15a17244 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244 2024-12-02T13:19:19,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T13:19:19,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e2885e920c8246a9a28fac9864ff61fd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd 2024-12-02T13:19:19,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T13:19:19,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for a6dac54182c9e8f0ab33f471f7aac1b1 in 959ms, sequenceid=11, compaction requested=false 2024-12-02T13:19:19,590 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T13:19:19,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:19,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T13:19:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:19,691 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-02T13:19:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:19,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/189f8a81600b491abeb707d7f50899ea is 50, key is test_row_0/A:col10/1733145558728/Put/seqid=0 2024-12-02T13:19:19,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742298_1474 (size=12001) 2024-12-02T13:19:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:19,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145619880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145619886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145619894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145619894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145619903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145619994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145620003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,101 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/189f8a81600b491abeb707d7f50899ea 2024-12-02T13:19:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ffcd2d4a824747b0b81ec1608f89f721 is 50, key is test_row_0/B:col10/1733145558728/Put/seqid=0 2024-12-02T13:19:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742299_1475 (size=12001) 2024-12-02T13:19:20,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145620214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145620224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,513 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ffcd2d4a824747b0b81ec1608f89f721 2024-12-02T13:19:20,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145620525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec6901d23bca4a63a3ab389dbaedef1b is 50, key is test_row_0/C:col10/1733145558728/Put/seqid=0 2024-12-02T13:19:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145620535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742300_1476 (size=12001) 2024-12-02T13:19:20,572 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec6901d23bca4a63a3ab389dbaedef1b 2024-12-02T13:19:20,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/189f8a81600b491abeb707d7f50899ea as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea 2024-12-02T13:19:20,583 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:19:20,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ffcd2d4a824747b0b81ec1608f89f721 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721 2024-12-02T13:19:20,592 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:19:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec6901d23bca4a63a3ab389dbaedef1b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b 2024-12-02T13:19:20,596 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b, entries=150, sequenceid=47, filesize=11.7 K 2024-12-02T13:19:20,597 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for a6dac54182c9e8f0ab33f471f7aac1b1 in 906ms, sequenceid=47, compaction requested=false 2024-12-02T13:19:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T13:19:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T13:19:20,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T13:19:20,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-12-02T13:19:20,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.9850 sec 2024-12-02T13:19:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T13:19:20,764 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:20,764 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-02T13:19:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T13:19:20,766 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:20,767 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:20,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T13:19:20,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:20,919 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:20,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:20,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/7143ecde6ffa4ae6b9cd28a6fc69b725 is 50, key is test_row_0/A:col10/1733145559884/Put/seqid=0 2024-12-02T13:19:20,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742301_1477 (size=12001) 2024-12-02T13:19:20,956 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/7143ecde6ffa4ae6b9cd28a6fc69b725 2024-12-02T13:19:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/51d42fdb0721490ea00d53cf4aef556d is 50, key is test_row_0/B:col10/1733145559884/Put/seqid=0 2024-12-02T13:19:21,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742302_1478 (size=12001) 2024-12-02T13:19:21,003 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/51d42fdb0721490ea00d53cf4aef556d 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145560986, max timestamp=1733145561009Num uniques 859; 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145560986, max timestamp=1733145561009Num uniques 859; 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145560986, max timestamp=1733145561009Num uniques 859; 2024-12-02T13:19:21,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:21,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1f0e1ce5957f4816ac47f01143fa6f5d is 50, key is test_row_0/C:col10/1733145559884/Put/seqid=0 2024-12-02T13:19:21,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742303_1479 (size=12001) 2024-12-02T13:19:21,053 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1f0e1ce5957f4816ac47f01143fa6f5d 2024-12-02T13:19:21,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/7143ecde6ffa4ae6b9cd28a6fc69b725 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725 2024-12-02T13:19:21,073 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725, entries=150, sequenceid=56, filesize=11.7 K 2024-12-02T13:19:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/51d42fdb0721490ea00d53cf4aef556d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d 2024-12-02T13:19:21,079 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d, entries=150, sequenceid=56, filesize=11.7 K 2024-12-02T13:19:21,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1f0e1ce5957f4816ac47f01143fa6f5d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d 2024-12-02T13:19:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T13:19:21,084 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d, entries=150, sequenceid=56, filesize=11.7 K 2024-12-02T13:19:21,085 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=114.05 KB/116790 for a6dac54182c9e8f0ab33f471f7aac1b1 in 166ms, sequenceid=56, compaction requested=true 2024-12-02T13:19:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T13:19:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T13:19:21,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T13:19:21,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 320 msec 2024-12-02T13:19:21,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 327 msec 2024-12-02T13:19:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:21,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:21,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:21,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/17c76f3031de4f77863cf1cd0ae7f9a6 is 50, key is test_row_0/A:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742304_1480 (size=14341) 2024-12-02T13:19:21,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/17c76f3031de4f77863cf1cd0ae7f9a6 2024-12-02T13:19:21,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/1910caf021f54183a06a060d0116f08c is 50, key is test_row_0/B:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145621138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145621136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145621166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145621169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145621170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742305_1481 (size=12001) 2024-12-02T13:19:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145621274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145621274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145621294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145621305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145621305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T13:19:21,394 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:21,394 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-02T13:19:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:21,398 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:21,399 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:21,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145621484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145621484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:21,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145621513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,523 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:19:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145621524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145621524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T13:19:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:21,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/1910caf021f54183a06a060d0116f08c 2024-12-02T13:19:21,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0bd2c5bff26b4794b50aaa79666a6677 is 50, key is test_row_0/C:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742306_1482 (size=12001) 2024-12-02T13:19:21,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0bd2c5bff26b4794b50aaa79666a6677 2024-12-02T13:19:21,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/17c76f3031de4f77863cf1cd0ae7f9a6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6 2024-12-02T13:19:21,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6, entries=200, sequenceid=89, filesize=14.0 K 2024-12-02T13:19:21,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/1910caf021f54183a06a060d0116f08c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c 2024-12-02T13:19:21,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c, entries=150, sequenceid=89, filesize=11.7 K 2024-12-02T13:19:21,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0bd2c5bff26b4794b50aaa79666a6677 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677 2024-12-02T13:19:21,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677, entries=150, sequenceid=89, filesize=11.7 K 2024-12-02T13:19:21,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=67.09 KB/68700 for a6dac54182c9e8f0ab33f471f7aac1b1 in 543ms, sequenceid=89, compaction requested=true 2024-12-02T13:19:21,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:21,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:21,640 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:21,640 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:21,641 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:21,641 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:21,641 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,642 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=49.2 K 2024-12-02T13:19:21,642 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb34dd8fc034d09924e3b756ddc92e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733145558624 2024-12-02T13:19:21,642 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 189f8a81600b491abeb707d7f50899ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145558695 2024-12-02T13:19:21,642 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:21,642 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:21,642 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,642 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=46.9 K 2024-12-02T13:19:21,642 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7143ecde6ffa4ae6b9cd28a6fc69b725, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733145559877 2024-12-02T13:19:21,643 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c76f3031de4f77863cf1cd0ae7f9a6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:21,645 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a8690292af433fb1f0d2fe15a17244, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733145558624 2024-12-02T13:19:21,646 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ffcd2d4a824747b0b81ec1608f89f721, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145558695 2024-12-02T13:19:21,649 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d42fdb0721490ea00d53cf4aef556d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733145559877 2024-12-02T13:19:21,649 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1910caf021f54183a06a060d0116f08c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:21,653 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:21,653 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/703c0370b9f24563bbac639c4d427691 is 50, key is test_row_0/A:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,656 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:21,656 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ece55b8455a341a5bf89f71b732e7339 is 50, key is test_row_0/B:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742307_1483 (size=12139) 2024-12-02T13:19:21,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742308_1484 (size=12139) 2024-12-02T13:19:21,678 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ece55b8455a341a5bf89f71b732e7339 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ece55b8455a341a5bf89f71b732e7339 2024-12-02T13:19:21,684 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into ece55b8455a341a5bf89f71b732e7339(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:21,684 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:21,684 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=12, startTime=1733145561640; duration=0sec 2024-12-02T13:19:21,684 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:21,684 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:21,684 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:21,685 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:21,685 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:21,685 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,685 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=46.9 K 2024-12-02T13:19:21,685 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e2885e920c8246a9a28fac9864ff61fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733145558624 2024-12-02T13:19:21,685 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ec6901d23bca4a63a3ab389dbaedef1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733145558695 2024-12-02T13:19:21,686 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0e1ce5957f4816ac47f01143fa6f5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733145559877 2024-12-02T13:19:21,686 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd2c5bff26b4794b50aaa79666a6677, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:21,693 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:21,693 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0eb211053ac24007aaf25048b0ee749e is 50, key is test_row_0/C:col10/1733145561095/Put/seqid=0 2024-12-02T13:19:21,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742309_1485 (size=12139) 2024-12-02T13:19:21,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T13:19:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:21,712 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:19:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:21,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 is 50, key is test_row_0/A:col10/1733145561101/Put/seqid=0 2024-12-02T13:19:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742310_1486 (size=12001) 2024-12-02T13:19:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:21,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145621843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145621844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145621844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145621846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145621847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145621954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145621954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145621964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:22,075 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/703c0370b9f24563bbac639c4d427691 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/703c0370b9f24563bbac639c4d427691 2024-12-02T13:19:22,079 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 703c0370b9f24563bbac639c4d427691(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:22,079 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:22,079 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=12, startTime=1733145561639; duration=0sec 2024-12-02T13:19:22,079 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:22,079 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:22,102 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0eb211053ac24007aaf25048b0ee749e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0eb211053ac24007aaf25048b0ee749e 2024-12-02T13:19:22,106 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into 0eb211053ac24007aaf25048b0ee749e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:22,106 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:22,106 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=12, startTime=1733145561640; duration=0sec 2024-12-02T13:19:22,106 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:22,106 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:22,120 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 2024-12-02T13:19:22,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a2297995d3bc40a99aec8b171f7c35b3 is 50, key is test_row_0/B:col10/1733145561101/Put/seqid=0 2024-12-02T13:19:22,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742311_1487 (size=12001) 2024-12-02T13:19:22,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145622174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145622174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145622174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145622353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145622353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145622483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145622484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145622499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:22,531 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a2297995d3bc40a99aec8b171f7c35b3 2024-12-02T13:19:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/eff62e8145d444f7890d4b0721df3f60 is 50, key is test_row_0/C:col10/1733145561101/Put/seqid=0 2024-12-02T13:19:22,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742312_1488 (size=12001) 2024-12-02T13:19:22,555 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/eff62e8145d444f7890d4b0721df3f60 2024-12-02T13:19:22,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 2024-12-02T13:19:22,568 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334, entries=150, sequenceid=103, filesize=11.7 K 2024-12-02T13:19:22,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a2297995d3bc40a99aec8b171f7c35b3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3 2024-12-02T13:19:22,572 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3, entries=150, sequenceid=103, filesize=11.7 K 2024-12-02T13:19:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/eff62e8145d444f7890d4b0721df3f60 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60 2024-12-02T13:19:22,577 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60, entries=150, sequenceid=103, filesize=11.7 K 2024-12-02T13:19:22,578 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6dac54182c9e8f0ab33f471f7aac1b1 in 866ms, sequenceid=103, compaction requested=false 2024-12-02T13:19:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:22,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T13:19:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T13:19:22,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T13:19:22,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-12-02T13:19:22,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.1880 sec 2024-12-02T13:19:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:23,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/430627a1fa7d4cbf84cccc89e1e78712 is 50, key is test_row_0/A:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:23,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742313_1489 (size=14341) 2024-12-02T13:19:23,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/430627a1fa7d4cbf84cccc89e1e78712 2024-12-02T13:19:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145623017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145623025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/72bb52e79afe4aa591f51841cc78a342 is 50, key is test_row_0/B:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145623027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742314_1490 (size=12001) 2024-12-02T13:19:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145623134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145623134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145623144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145623344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145623344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145623354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145623363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,369 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145623373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,377 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/72bb52e79afe4aa591f51841cc78a342 2024-12-02T13:19:23,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ce2efba894864ab19e2cba29a3caa4fa is 50, key is test_row_0/C:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:23,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742315_1491 (size=12001) 2024-12-02T13:19:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T13:19:23,544 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:23,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:23,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-02T13:19:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:23,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:23,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:23,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:23,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145623654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145623654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145623665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:23,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T13:19:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T13:19:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:23,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:23,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ce2efba894864ab19e2cba29a3caa4fa 2024-12-02T13:19:23,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/430627a1fa7d4cbf84cccc89e1e78712 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712 2024-12-02T13:19:23,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712, entries=200, sequenceid=125, filesize=14.0 K 2024-12-02T13:19:23,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/72bb52e79afe4aa591f51841cc78a342 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342 2024-12-02T13:19:23,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T13:19:23,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ce2efba894864ab19e2cba29a3caa4fa as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa 2024-12-02T13:19:23,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T13:19:23,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6dac54182c9e8f0ab33f471f7aac1b1 in 891ms, sequenceid=125, compaction requested=true 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:23,891 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:23,891 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:23,892 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:23,892 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:23,892 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,893 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ece55b8455a341a5bf89f71b732e7339, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=35.3 K 2024-12-02T13:19:23,893 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:23,893 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:23,893 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,893 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/703c0370b9f24563bbac639c4d427691, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=37.6 K 2024-12-02T13:19:23,893 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703c0370b9f24563bbac639c4d427691, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:23,893 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ece55b8455a341a5bf89f71b732e7339, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:23,894 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e0ef0d2beb543f4aa7b5fa2bedb3334, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733145561101 2024-12-02T13:19:23,894 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a2297995d3bc40a99aec8b171f7c35b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733145561101 2024-12-02T13:19:23,894 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 430627a1fa7d4cbf84cccc89e1e78712, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561807 2024-12-02T13:19:23,894 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bb52e79afe4aa591f51841cc78a342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561842 2024-12-02T13:19:23,902 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:23,903 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/a702d16921a64bf9b2b94e7684038405 is 50, key is test_row_0/A:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:23,905 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:23,906 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/7441317c1da54e3897220ae5af673f1d is 50, key is test_row_0/B:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:23,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742317_1493 (size=12241) 2024-12-02T13:19:23,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742316_1492 (size=12241) 2024-12-02T13:19:23,953 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/a702d16921a64bf9b2b94e7684038405 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/a702d16921a64bf9b2b94e7684038405 2024-12-02T13:19:23,957 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into a702d16921a64bf9b2b94e7684038405(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:23,957 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:23,957 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145563891; duration=0sec 2024-12-02T13:19:23,958 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:23,958 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:23,958 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:23,959 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:23,960 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:23,960 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:23,960 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0eb211053ac24007aaf25048b0ee749e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=35.3 K 2024-12-02T13:19:23,960 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb211053ac24007aaf25048b0ee749e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733145561043 2024-12-02T13:19:23,960 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting eff62e8145d444f7890d4b0721df3f60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733145561101 2024-12-02T13:19:23,961 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2efba894864ab19e2cba29a3caa4fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561842 2024-12-02T13:19:23,975 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:23,976 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/c13f85eebef34417b2397a5dcb42af61 is 50, key is test_row_0/C:col10/1733145562996/Put/seqid=0 2024-12-02T13:19:24,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742318_1494 (size=12241) 2024-12-02T13:19:24,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T13:19:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:24,007 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:19:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:24,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:24,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/6d08648db6eb4759a39c832ebffd8f0a is 50, key is test_row_0/A:col10/1733145563024/Put/seqid=0 2024-12-02T13:19:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742319_1495 (size=12151) 2024-12-02T13:19:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:24,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:24,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145564202, max timestamp=1733145564229Num uniques 859; 2024-12-02T13:19:24,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145564202, max timestamp=1733145564229Num uniques 859; 2024-12-02T13:19:24,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:24,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:24,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145564202, max timestamp=1733145564229Num uniques 859; 2024-12-02T13:19:24,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:24,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145624272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145624284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145624285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,350 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/7441317c1da54e3897220ae5af673f1d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/7441317c1da54e3897220ae5af673f1d 2024-12-02T13:19:24,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into 7441317c1da54e3897220ae5af673f1d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:24,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:24,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145563891; duration=0sec 2024-12-02T13:19:24,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:24,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145624393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,409 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/c13f85eebef34417b2397a5dcb42af61 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c13f85eebef34417b2397a5dcb42af61 2024-12-02T13:19:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145624404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145624404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,415 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into c13f85eebef34417b2397a5dcb42af61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:24,415 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:24,415 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145563891; duration=0sec 2024-12-02T13:19:24,415 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:24,415 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:24,427 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/6d08648db6eb4759a39c832ebffd8f0a 2024-12-02T13:19:24,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/cdd491d696bd4398ab91c14e783026df is 50, key is test_row_0/B:col10/1733145563024/Put/seqid=0 2024-12-02T13:19:24,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742320_1496 (size=12151) 2024-12-02T13:19:24,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145624614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145624614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145624614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:24,853 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/cdd491d696bd4398ab91c14e783026df 2024-12-02T13:19:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/29a3a1a0b9044f63b53a15cd7aba02bb is 50, key is test_row_0/C:col10/1733145563024/Put/seqid=0 2024-12-02T13:19:24,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742321_1497 (size=12151) 2024-12-02T13:19:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145624924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145624924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:24,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145624933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,262 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/29a3a1a0b9044f63b53a15cd7aba02bb 2024-12-02T13:19:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/6d08648db6eb4759a39c832ebffd8f0a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a 2024-12-02T13:19:25,269 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a, entries=150, sequenceid=138, filesize=11.9 K 2024-12-02T13:19:25,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/cdd491d696bd4398ab91c14e783026df as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df 2024-12-02T13:19:25,272 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df, entries=150, sequenceid=138, filesize=11.9 K 2024-12-02T13:19:25,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/29a3a1a0b9044f63b53a15cd7aba02bb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb 2024-12-02T13:19:25,275 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb, entries=150, sequenceid=138, filesize=11.9 K 2024-12-02T13:19:25,276 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for a6dac54182c9e8f0ab33f471f7aac1b1 in 1269ms, sequenceid=138, compaction requested=false 2024-12-02T13:19:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:25,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-02T13:19:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-02T13:19:25,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T13:19:25,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7310 sec 2024-12-02T13:19:25,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.7350 sec 2024-12-02T13:19:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:25,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:25,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:25,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0abf3cce846043b18660191291ff8e4d is 50, key is test_row_0/A:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:25,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742322_1498 (size=16931) 2024-12-02T13:19:25,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145625423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145625426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145625444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145625444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145625453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145625533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145625533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T13:19:25,704 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:25,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-12-02T13:19:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:25,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:25,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:25,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145625744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145625744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:25,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0abf3cce846043b18660191291ff8e4d 2024-12-02T13:19:25,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/c6c4f36cab394c29a60b64ac108a9a58 is 50, key is test_row_0/B:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:25,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742323_1499 (size=12151) 2024-12-02T13:19:25,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:25,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:25,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:25,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:25,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:26,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:26,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145626054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145626064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/c6c4f36cab394c29a60b64ac108a9a58 2024-12-02T13:19:26,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/336ad6355b6942f3b9969a4e58ccbd8f is 50, key is test_row_0/C:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:26,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742324_1500 (size=12151) 2024-12-02T13:19:26,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:26,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145626463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145626463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,469 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:26,469 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145626474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,485 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145626573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145626584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:26,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:26,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/336ad6355b6942f3b9969a4e58ccbd8f 2024-12-02T13:19:26,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0abf3cce846043b18660191291ff8e4d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d 2024-12-02T13:19:26,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d, entries=250, sequenceid=176, filesize=16.5 K 2024-12-02T13:19:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/c6c4f36cab394c29a60b64ac108a9a58 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58 2024-12-02T13:19:26,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58, entries=150, sequenceid=176, filesize=11.9 K 2024-12-02T13:19:26,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/336ad6355b6942f3b9969a4e58ccbd8f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f 2024-12-02T13:19:26,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f, entries=150, sequenceid=176, filesize=11.9 K 2024-12-02T13:19:26,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a6dac54182c9e8f0ab33f471f7aac1b1 in 1254ms, sequenceid=176, compaction requested=true 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:26,653 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:26,653 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:26,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:26,654 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,654 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/7441317c1da54e3897220ae5af673f1d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=35.7 K 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7441317c1da54e3897220ae5af673f1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561842 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:26,654 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,654 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/a702d16921a64bf9b2b94e7684038405, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=40.4 K 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd491d696bd4398ab91c14e783026df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733145563013 2024-12-02T13:19:26,654 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a702d16921a64bf9b2b94e7684038405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561842 2024-12-02T13:19:26,655 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d08648db6eb4759a39c832ebffd8f0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733145563013 2024-12-02T13:19:26,655 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c4f36cab394c29a60b64ac108a9a58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564284 2024-12-02T13:19:26,655 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0abf3cce846043b18660191291ff8e4d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564266 2024-12-02T13:19:26,664 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:26,665 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/93cd65b1c5cb4dffa966f8dc9ada56b6 is 50, key is test_row_0/A:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:26,667 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:26,667 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/768bd00ab15e4089a2e7068a894fa2a5 is 50, key is test_row_0/B:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:26,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742325_1501 (size=12493) 2024-12-02T13:19:26,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742326_1502 (size=12493) 2024-12-02T13:19:26,676 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/93cd65b1c5cb4dffa966f8dc9ada56b6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/93cd65b1c5cb4dffa966f8dc9ada56b6 2024-12-02T13:19:26,682 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 93cd65b1c5cb4dffa966f8dc9ada56b6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:26,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:26,682 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145566653; duration=0sec 2024-12-02T13:19:26,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:26,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:26,682 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/768bd00ab15e4089a2e7068a894fa2a5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/768bd00ab15e4089a2e7068a894fa2a5 2024-12-02T13:19:26,682 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:26,683 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:26,683 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:26,683 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,683 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c13f85eebef34417b2397a5dcb42af61, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=35.7 K 2024-12-02T13:19:26,683 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c13f85eebef34417b2397a5dcb42af61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733145561842 2024-12-02T13:19:26,683 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29a3a1a0b9044f63b53a15cd7aba02bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733145563013 2024-12-02T13:19:26,684 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336ad6355b6942f3b9969a4e58ccbd8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564284 2024-12-02T13:19:26,686 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into 768bd00ab15e4089a2e7068a894fa2a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:26,686 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:26,686 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145566653; duration=0sec 2024-12-02T13:19:26,686 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:26,686 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:26,690 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:26,690 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/6a480e87f7bd4634b1a9cb3fa4c2c318 is 50, key is test_row_0/C:col10/1733145565397/Put/seqid=0 2024-12-02T13:19:26,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742327_1503 (size=12493) 2024-12-02T13:19:26,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-12-02T13:19:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:26,774 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:26,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:26,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/ddcf4bc74c1e48e3a9baaa67b86f03bf is 50, key is test_row_0/A:col10/1733145565415/Put/seqid=0 2024-12-02T13:19:26,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742328_1504 (size=12151) 2024-12-02T13:19:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:27,101 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/6a480e87f7bd4634b1a9cb3fa4c2c318 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6a480e87f7bd4634b1a9cb3fa4c2c318 2024-12-02T13:19:27,105 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into 6a480e87f7bd4634b1a9cb3fa4c2c318(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:27,105 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:27,105 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145566653; duration=0sec 2024-12-02T13:19:27,105 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:27,105 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:27,181 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/ddcf4bc74c1e48e3a9baaa67b86f03bf 2024-12-02T13:19:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e7fdede82a2f4257993070d1cdc6ec3b is 50, key is test_row_0/B:col10/1733145565415/Put/seqid=0 2024-12-02T13:19:27,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742329_1505 (size=12151) 2024-12-02T13:19:27,190 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e7fdede82a2f4257993070d1cdc6ec3b 2024-12-02T13:19:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8cabbb7431dc4f71b18a34a721dac1af is 50, key is test_row_0/C:col10/1733145565415/Put/seqid=0 2024-12-02T13:19:27,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742330_1506 (size=12151) 2024-12-02T13:19:27,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:27,597 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8cabbb7431dc4f71b18a34a721dac1af 2024-12-02T13:19:27,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/ddcf4bc74c1e48e3a9baaa67b86f03bf as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf 2024-12-02T13:19:27,604 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf, entries=150, sequenceid=188, filesize=11.9 K 2024-12-02T13:19:27,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e7fdede82a2f4257993070d1cdc6ec3b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b 2024-12-02T13:19:27,607 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b, entries=150, sequenceid=188, filesize=11.9 K 2024-12-02T13:19:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8cabbb7431dc4f71b18a34a721dac1af as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af 2024-12-02T13:19:27,611 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af, entries=150, sequenceid=188, filesize=11.9 K 2024-12-02T13:19:27,612 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=46.96 KB/48090 for a6dac54182c9e8f0ab33f471f7aac1b1 in 838ms, sequenceid=188, compaction requested=false 2024-12-02T13:19:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-02T13:19:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-02T13:19:27,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T13:19:27,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9070 sec 2024-12-02T13:19:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:27,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:27,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:27,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.9110 sec 2024-12-02T13:19:27,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d49c583c42844e088e1eeb369244dbb0 is 50, key is test_row_0/A:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:27,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742331_1507 (size=12147) 2024-12-02T13:19:27,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d49c583c42844e088e1eeb369244dbb0 2024-12-02T13:19:27,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a80f2c73e0f94dc69d8ef4fac0475148 is 50, key is test_row_0/B:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742332_1508 (size=9757) 2024-12-02T13:19:27,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:27,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145567650, max timestamp=1733145567671Num uniques 859; 2024-12-02T13:19:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145567650, max timestamp=1733145567671Num uniques 859; 2024-12-02T13:19:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145627666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145567650, max timestamp=1733145567671Num uniques 859; 2024-12-02T13:19:27,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:27,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145627747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:27,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145627784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T13:19:27,853 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:27,854 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-12-02T13:19:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:27,855 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:27,856 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:27,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145627854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:28,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145627994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,005 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-02T13:19:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a80f2c73e0f94dc69d8ef4fac0475148 2024-12-02T13:19:28,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/388c3af82f1f4471a1d919bba6070486 is 50, key is test_row_0/C:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742333_1509 (size=9757) 2024-12-02T13:19:28,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145628073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-02T13:19:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:28,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-02T13:19:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:28,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145628313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145628384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/388c3af82f1f4471a1d919bba6070486 2024-12-02T13:19:28,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d49c583c42844e088e1eeb369244dbb0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0 2024-12-02T13:19:28,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0, entries=150, sequenceid=200, filesize=11.9 K 2024-12-02T13:19:28,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a80f2c73e0f94dc69d8ef4fac0475148 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148 2024-12-02T13:19:28,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148, entries=100, sequenceid=200, filesize=9.5 K 2024-12-02T13:19:28,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/388c3af82f1f4471a1d919bba6070486 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486 2024-12-02T13:19:28,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486, entries=100, sequenceid=200, filesize=9.5 K 2024-12-02T13:19:28,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a6dac54182c9e8f0ab33f471f7aac1b1 in 837ms, sequenceid=200, compaction requested=true 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:28,452 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:28,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:28,452 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:28,453 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,453 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,453 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/768bd00ab15e4089a2e7068a894fa2a5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=33.6 K 2024-12-02T13:19:28,453 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/93cd65b1c5cb4dffa966f8dc9ada56b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=35.9 K 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 768bd00ab15e4089a2e7068a894fa2a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564284 2024-12-02T13:19:28,453 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93cd65b1c5cb4dffa966f8dc9ada56b6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564284 2024-12-02T13:19:28,454 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e7fdede82a2f4257993070d1cdc6ec3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733145565415 2024-12-02T13:19:28,454 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddcf4bc74c1e48e3a9baaa67b86f03bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733145565415 2024-12-02T13:19:28,454 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a80f2c73e0f94dc69d8ef4fac0475148, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145567611 2024-12-02T13:19:28,454 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d49c583c42844e088e1eeb369244dbb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145567608 2024-12-02T13:19:28,460 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:28,460 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/aaa5f489b06642bca2050b55d285a866 is 50, key is test_row_0/B:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:28,460 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:28,461 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/4df46a53434e46dd8a5bf9cc6840a1fd is 50, key is test_row_0/A:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:28,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,465 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:28,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b929b7590d474d26b704c420ca0b439b is 50, key is test_row_0/A:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742335_1511 (size=12595) 2024-12-02T13:19:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:28,488 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/4df46a53434e46dd8a5bf9cc6840a1fd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/4df46a53434e46dd8a5bf9cc6840a1fd 2024-12-02T13:19:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:28,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:28,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742334_1510 (size=12595) 2024-12-02T13:19:28,496 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 4df46a53434e46dd8a5bf9cc6840a1fd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:28,496 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:28,496 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145568452; duration=0sec 2024-12-02T13:19:28,496 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:28,496 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:28,496 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:28,497 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:28,497 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:28,497 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:28,497 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6a480e87f7bd4634b1a9cb3fa4c2c318, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=33.6 K 2024-12-02T13:19:28,499 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a480e87f7bd4634b1a9cb3fa4c2c318, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733145564284 2024-12-02T13:19:28,500 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cabbb7431dc4f71b18a34a721dac1af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733145565415 2024-12-02T13:19:28,500 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 388c3af82f1f4471a1d919bba6070486, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145567611 2024-12-02T13:19:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742336_1512 (size=12151) 2024-12-02T13:19:28,506 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:28,506 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/6742ede8d9094fb1aeec092ef9e2f153 is 50, key is test_row_0/C:col10/1733145567611/Put/seqid=0 2024-12-02T13:19:28,507 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b929b7590d474d26b704c420ca0b439b 2024-12-02T13:19:28,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742337_1513 (size=12595) 2024-12-02T13:19:28,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ba202c8dfa3a488588919115f94f4d81 is 50, key is test_row_0/B:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742338_1514 (size=12151) 2024-12-02T13:19:28,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145628510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145628522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145628522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145628634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145628634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145628634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145628823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145628844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145628844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145628844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,898 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/aaa5f489b06642bca2050b55d285a866 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/aaa5f489b06642bca2050b55d285a866 2024-12-02T13:19:28,903 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into aaa5f489b06642bca2050b55d285a866(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:28,903 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:28,903 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145568452; duration=0sec 2024-12-02T13:19:28,903 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:28,903 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145628903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:28,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:28,922 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ba202c8dfa3a488588919115f94f4d81 2024-12-02T13:19:28,923 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/6742ede8d9094fb1aeec092ef9e2f153 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6742ede8d9094fb1aeec092ef9e2f153 2024-12-02T13:19:28,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/447372a819bb4b9c84807e023ad5bf0e is 50, key is test_row_0/C:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:28,939 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into 6742ede8d9094fb1aeec092ef9e2f153(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:28,939 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:28,939 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145568452; duration=0sec 2024-12-02T13:19:28,939 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:28,939 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:28,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742339_1515 (size=12151) 2024-12-02T13:19:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145629153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145629154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145629164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,346 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/447372a819bb4b9c84807e023ad5bf0e 2024-12-02T13:19:29,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b929b7590d474d26b704c420ca0b439b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b 2024-12-02T13:19:29,353 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b, entries=150, sequenceid=235, filesize=11.9 K 2024-12-02T13:19:29,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ba202c8dfa3a488588919115f94f4d81 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81 2024-12-02T13:19:29,356 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81, entries=150, sequenceid=235, filesize=11.9 K 2024-12-02T13:19:29,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/447372a819bb4b9c84807e023ad5bf0e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e 2024-12-02T13:19:29,359 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e, entries=150, sequenceid=235, filesize=11.9 K 2024-12-02T13:19:29,360 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a6dac54182c9e8f0ab33f471f7aac1b1 in 895ms, sequenceid=235, compaction requested=false 2024-12-02T13:19:29,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:29,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:29,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-02T13:19:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-02T13:19:29,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-02T13:19:29,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5050 sec 2024-12-02T13:19:29,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.5090 sec 2024-12-02T13:19:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:29,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:29,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:29,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/5541791cde40466d92ebfdfbb4db2841 is 50, key is test_row_1/A:col10/1733145569674/Put/seqid=0 2024-12-02T13:19:29,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742340_1516 (size=12147) 2024-12-02T13:19:29,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/5541791cde40466d92ebfdfbb4db2841 2024-12-02T13:19:29,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/880bf9264b274e3b8887a88bdd4d2bd5 is 50, key is test_row_1/B:col10/1733145569674/Put/seqid=0 2024-12-02T13:19:29,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742341_1517 (size=9757) 2024-12-02T13:19:29,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145629723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145629724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145629725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145629834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145629844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145629844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145629844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,849 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:29,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145629924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:29,929 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T13:19:30,004 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:30,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-12-02T13:19:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:30,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:30,006 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:30,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:30,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145630043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145630053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:30,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/880bf9264b274e3b8887a88bdd4d2bd5 2024-12-02T13:19:30,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1df6c074de1b4617a178648524049d6a is 50, key is test_row_1/C:col10/1733145569674/Put/seqid=0 2024-12-02T13:19:30,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742342_1518 (size=9757) 2024-12-02T13:19:30,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T13:19:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:30,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T13:19:30,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:30,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:30,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145630353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145630363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145630363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T13:19:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:30,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:30,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1df6c074de1b4617a178648524049d6a 2024-12-02T13:19:30,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/5541791cde40466d92ebfdfbb4db2841 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841 2024-12-02T13:19:30,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841, entries=150, sequenceid=250, filesize=11.9 K 2024-12-02T13:19:30,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/880bf9264b274e3b8887a88bdd4d2bd5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5 2024-12-02T13:19:30,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5, entries=100, sequenceid=250, filesize=9.5 K 2024-12-02T13:19:30,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/1df6c074de1b4617a178648524049d6a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a 2024-12-02T13:19:30,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a, entries=100, sequenceid=250, filesize=9.5 K 2024-12-02T13:19:30,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a6dac54182c9e8f0ab33f471f7aac1b1 in 862ms, sequenceid=250, compaction requested=true 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:30,537 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:30,537 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:30,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36893 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:30,538 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,538 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,538 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/4df46a53434e46dd8a5bf9cc6840a1fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.0 K 2024-12-02T13:19:30,538 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/aaa5f489b06642bca2050b55d285a866, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=33.7 K 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df46a53434e46dd8a5bf9cc6840a1fd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145565425 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting aaa5f489b06642bca2050b55d285a866, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145565425 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b929b7590d474d26b704c420ca0b439b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733145567710 2024-12-02T13:19:30,538 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ba202c8dfa3a488588919115f94f4d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733145567710 2024-12-02T13:19:30,539 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5541791cde40466d92ebfdfbb4db2841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145568508 2024-12-02T13:19:30,539 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 880bf9264b274e3b8887a88bdd4d2bd5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145568521 2024-12-02T13:19:30,543 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:30,543 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:30,543 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/0835f1a050af4ba5b9b6e62df2b70aa2 is 50, key is test_row_0/B:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:30,543 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/73db88cf5f664ab2b62ede4bf30387c1 is 50, key is test_row_0/A:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:30,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742344_1520 (size=12697) 2024-12-02T13:19:30,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742343_1519 (size=12697) 2024-12-02T13:19:30,558 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/73db88cf5f664ab2b62ede4bf30387c1 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/73db88cf5f664ab2b62ede4bf30387c1 2024-12-02T13:19:30,561 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 73db88cf5f664ab2b62ede4bf30387c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:30,561 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:30,561 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145570537; duration=0sec 2024-12-02T13:19:30,561 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:30,561 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:30,561 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:30,562 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:30,563 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:30,563 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,563 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6742ede8d9094fb1aeec092ef9e2f153, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=33.7 K 2024-12-02T13:19:30,563 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6742ede8d9094fb1aeec092ef9e2f153, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733145565425 2024-12-02T13:19:30,563 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447372a819bb4b9c84807e023ad5bf0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733145567710 2024-12-02T13:19:30,563 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df6c074de1b4617a178648524049d6a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145568521 2024-12-02T13:19:30,568 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:30,568 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/dfbb4790b6a9436cbf1c9c457d1f41bc is 50, key is test_row_0/C:col10/1733145567710/Put/seqid=0 2024-12-02T13:19:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742345_1521 (size=12697) 2024-12-02T13:19:30,577 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/dfbb4790b6a9436cbf1c9c457d1f41bc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/dfbb4790b6a9436cbf1c9c457d1f41bc 2024-12-02T13:19:30,580 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into dfbb4790b6a9436cbf1c9c457d1f41bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:30,581 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:30,581 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145570537; duration=0sec 2024-12-02T13:19:30,581 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:30,581 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:30,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-12-02T13:19:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:30,616 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:19:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:30,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:30,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d2faa9f780d74297b7d4a223501a4e30 is 50, key is test_row_0/A:col10/1733145569723/Put/seqid=0 2024-12-02T13:19:30,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742346_1522 (size=12301) 2024-12-02T13:19:30,624 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d2faa9f780d74297b7d4a223501a4e30 2024-12-02T13:19:30,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/3b29d0897b1f445aa822a661607a0013 is 50, key is test_row_0/B:col10/1733145569723/Put/seqid=0 2024-12-02T13:19:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742347_1523 (size=12301) 2024-12-02T13:19:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:30,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:30,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145630882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145630883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145630888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/0835f1a050af4ba5b9b6e62df2b70aa2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/0835f1a050af4ba5b9b6e62df2b70aa2 2024-12-02T13:19:30,960 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into 0835f1a050af4ba5b9b6e62df2b70aa2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:30,960 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:30,960 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145570537; duration=0sec 2024-12-02T13:19:30,960 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:30,961 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145630994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145630994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145631004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,046 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/3b29d0897b1f445aa822a661607a0013 2024-12-02T13:19:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/a7c0eefe378443bb9812939b81c78ba3 is 50, key is test_row_0/C:col10/1733145569723/Put/seqid=0 2024-12-02T13:19:31,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742348_1524 (size=12301) 2024-12-02T13:19:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145631204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145631204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145631214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,455 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/a7c0eefe378443bb9812939b81c78ba3 2024-12-02T13:19:31,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/d2faa9f780d74297b7d4a223501a4e30 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30 2024-12-02T13:19:31,465 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30, entries=150, sequenceid=272, filesize=12.0 K 2024-12-02T13:19:31,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/3b29d0897b1f445aa822a661607a0013 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013 2024-12-02T13:19:31,468 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013, entries=150, sequenceid=272, filesize=12.0 K 2024-12-02T13:19:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/a7c0eefe378443bb9812939b81c78ba3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3 2024-12-02T13:19:31,472 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3, entries=150, sequenceid=272, filesize=12.0 K 2024-12-02T13:19:31,473 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6dac54182c9e8f0ab33f471f7aac1b1 in 856ms, sequenceid=272, compaction requested=false 2024-12-02T13:19:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-02T13:19:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-02T13:19:31,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T13:19:31,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4670 sec 2024-12-02T13:19:31,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.4710 sec 2024-12-02T13:19:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:31,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:19:31,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:31,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:31,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cb1392c72ce04cf4b367b092eb7b9772 is 50, key is test_row_0/A:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:31,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742349_1525 (size=12301) 2024-12-02T13:19:31,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cb1392c72ce04cf4b367b092eb7b9772 2024-12-02T13:19:31,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/8f99964a20034d6299b5657842375f6e is 50, key is test_row_0/B:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742350_1526 (size=12301) 2024-12-02T13:19:31,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/8f99964a20034d6299b5657842375f6e 2024-12-02T13:19:31,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/41b9f400d26f4807a3e98954a1c45ff8 is 50, key is test_row_0/C:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:31,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742351_1527 (size=12301) 2024-12-02T13:19:31,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145631558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145631559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145631560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145631664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145631664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145631673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145631874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145631874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145631874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,877 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145631883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/41b9f400d26f4807a3e98954a1c45ff8 2024-12-02T13:19:31,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cb1392c72ce04cf4b367b092eb7b9772 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772 2024-12-02T13:19:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145631944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:31,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772, entries=150, sequenceid=287, filesize=12.0 K 2024-12-02T13:19:31,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/8f99964a20034d6299b5657842375f6e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e 2024-12-02T13:19:31,949 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:31,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:31,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e, entries=150, sequenceid=287, filesize=12.0 K 2024-12-02T13:19:31,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/41b9f400d26f4807a3e98954a1c45ff8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8 2024-12-02T13:19:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8, entries=150, sequenceid=287, filesize=12.0 K 2024-12-02T13:19:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a6dac54182c9e8f0ab33f471f7aac1b1 in 438ms, sequenceid=287, compaction requested=true 2024-12-02T13:19:31,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:31,956 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:31,956 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:31,956 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:31,956 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:31,956 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:31,957 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/73db88cf5f664ab2b62ede4bf30387c1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.4 K 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:31,957 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73db88cf5f664ab2b62ede4bf30387c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145567710 2024-12-02T13:19:31,957 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/0835f1a050af4ba5b9b6e62df2b70aa2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.4 K 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2faa9f780d74297b7d4a223501a4e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733145569719 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0835f1a050af4ba5b9b6e62df2b70aa2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145567710 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1392c72ce04cf4b367b092eb7b9772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b29d0897b1f445aa822a661607a0013, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733145569719 2024-12-02T13:19:31,957 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f99964a20034d6299b5657842375f6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:31,962 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:31,962 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b6f6d304df2a404685f464ffad73bcf5 is 50, key is test_row_0/A:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:31,963 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:31,963 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/4c2827e38e01474c9c8614d667337752 is 50, key is test_row_0/B:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742352_1528 (size=12949) 2024-12-02T13:19:31,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742353_1529 (size=12949) 2024-12-02T13:19:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T13:19:32,154 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:32,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-12-02T13:19:32,157 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T13:19:32,158 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:32,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:32,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/aaea6b7087f34e66af84cdf34e327324 is 50, key is test_row_0/A:col10/1733145571557/Put/seqid=0 2024-12-02T13:19:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742354_1530 (size=14741) 2024-12-02T13:19:32,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/aaea6b7087f34e66af84cdf34e327324 2024-12-02T13:19:32,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/fededa3c72eb4dcea84cbf933524d094 is 50, key is test_row_0/B:col10/1733145571557/Put/seqid=0 2024-12-02T13:19:32,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742355_1531 (size=12301) 2024-12-02T13:19:32,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/fededa3c72eb4dcea84cbf933524d094 2024-12-02T13:19:32,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8298706c632c44f984196e68b8ce7339 is 50, key is test_row_0/C:col10/1733145571557/Put/seqid=0 2024-12-02T13:19:32,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742356_1532 (size=12301) 2024-12-02T13:19:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145632221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145632222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145632232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T13:19:32,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-02T13:19:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145632334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145632344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145632344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,368 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b6f6d304df2a404685f464ffad73bcf5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b6f6d304df2a404685f464ffad73bcf5 2024-12-02T13:19:32,371 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/4c2827e38e01474c9c8614d667337752 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/4c2827e38e01474c9c8614d667337752 2024-12-02T13:19:32,373 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into b6f6d304df2a404685f464ffad73bcf5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:32,373 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:32,373 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145571956; duration=0sec 2024-12-02T13:19:32,373 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:32,373 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:32,374 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:32,374 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:32,375 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into 4c2827e38e01474c9c8614d667337752(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:32,375 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145571956; duration=0sec 2024-12-02T13:19:32,375 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:32,375 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/dfbb4790b6a9436cbf1c9c457d1f41bc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.4 K 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfbb4790b6a9436cbf1c9c457d1f41bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733145567710 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7c0eefe378443bb9812939b81c78ba3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733145569719 2024-12-02T13:19:32,375 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b9f400d26f4807a3e98954a1c45ff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:32,379 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:32,380 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/66617f7b6dd64e17b4e154764124aa77 is 50, key is test_row_0/C:col10/1733145570887/Put/seqid=0 2024-12-02T13:19:32,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742357_1533 (size=12949) 2024-12-02T13:19:32,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-02T13:19:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:32,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T13:19:32,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145632544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145632563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145632563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-02T13:19:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:32,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8298706c632c44f984196e68b8ce7339 2024-12-02T13:19:32,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/aaea6b7087f34e66af84cdf34e327324 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324 2024-12-02T13:19:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324, entries=200, sequenceid=306, filesize=14.4 K 2024-12-02T13:19:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/fededa3c72eb4dcea84cbf933524d094 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094 2024-12-02T13:19:32,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094, entries=150, sequenceid=306, filesize=12.0 K 2024-12-02T13:19:32,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8298706c632c44f984196e68b8ce7339 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339 2024-12-02T13:19:32,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339, entries=150, sequenceid=306, filesize=12.0 K 2024-12-02T13:19:32,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a6dac54182c9e8f0ab33f471f7aac1b1 in 448ms, sequenceid=306, compaction requested=false 2024-12-02T13:19:32,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:32,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-12-02T13:19:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:32,766 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:32,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/695d3230d8cf4a2da3a7088468d30ca7 is 50, key is test_row_0/A:col10/1733145572230/Put/seqid=0 2024-12-02T13:19:32,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742358_1534 (size=12301) 2024-12-02T13:19:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T13:19:32,786 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/66617f7b6dd64e17b4e154764124aa77 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/66617f7b6dd64e17b4e154764124aa77 2024-12-02T13:19:32,790 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into 66617f7b6dd64e17b4e154764124aa77(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:32,790 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:32,790 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145571956; duration=0sec 2024-12-02T13:19:32,790 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:32,790 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:32,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145632893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145632896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:32,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145632897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145633004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145633004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145633013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,173 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/695d3230d8cf4a2da3a7088468d30ca7 2024-12-02T13:19:33,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e5a8955c51e145039e1085b1822b604c is 50, key is test_row_0/B:col10/1733145572230/Put/seqid=0 2024-12-02T13:19:33,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742359_1535 (size=12301) 2024-12-02T13:19:33,193 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e5a8955c51e145039e1085b1822b604c 2024-12-02T13:19:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/2585b6c6c09d44819e528c1a4b760dd6 is 50, key is test_row_0/C:col10/1733145572230/Put/seqid=0 2024-12-02T13:19:33,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742360_1536 (size=12301) 2024-12-02T13:19:33,219 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/2585b6c6c09d44819e528c1a4b760dd6 2024-12-02T13:19:33,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145633214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145633214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/695d3230d8cf4a2da3a7088468d30ca7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7 2024-12-02T13:19:33,228 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7, entries=150, sequenceid=321, filesize=12.0 K 2024-12-02T13:19:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145633223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/e5a8955c51e145039e1085b1822b604c as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c 2024-12-02T13:19:33,234 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c, entries=150, sequenceid=321, filesize=12.0 K 2024-12-02T13:19:33,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/2585b6c6c09d44819e528c1a4b760dd6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6 2024-12-02T13:19:33,238 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6, entries=150, sequenceid=321, filesize=12.0 K 2024-12-02T13:19:33,239 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6dac54182c9e8f0ab33f471f7aac1b1 in 473ms, sequenceid=321, compaction requested=true 2024-12-02T13:19:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-12-02T13:19:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-12-02T13:19:33,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-02T13:19:33,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0820 sec 2024-12-02T13:19:33,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.0880 sec 2024-12-02T13:19:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T13:19:33,295 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:33,295 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-12-02T13:19:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T13:19:33,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:33,298 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:33,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T13:19:33,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:33,450 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:33,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b65cadcd8e26428db63b7cbd415edb19 is 50, key is test_row_0/A:col10/1733145572895/Put/seqid=0 2024-12-02T13:19:33,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742361_1537 (size=12301) 2024-12-02T13:19:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:33,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145633558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145633568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145633570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T13:19:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145633673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145633673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145633693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,867 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b65cadcd8e26428db63b7cbd415edb19 2024-12-02T13:19:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/654d020f997c46d3987ca55ef6218d87 is 50, key is test_row_0/B:col10/1733145572895/Put/seqid=0 2024-12-02T13:19:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742362_1538 (size=12301) 2024-12-02T13:19:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145633884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145633884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145633903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T13:19:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145634194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145634204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145634214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,276 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/654d020f997c46d3987ca55ef6218d87 2024-12-02T13:19:34,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec91e3e6095b4d5492d7753c42c58450 is 50, key is test_row_0/C:col10/1733145572895/Put/seqid=0 2024-12-02T13:19:34,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742363_1539 (size=12301) 2024-12-02T13:19:34,286 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec91e3e6095b4d5492d7753c42c58450 2024-12-02T13:19:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/b65cadcd8e26428db63b7cbd415edb19 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19 2024-12-02T13:19:34,291 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19, entries=150, sequenceid=341, filesize=12.0 K 2024-12-02T13:19:34,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/654d020f997c46d3987ca55ef6218d87 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87 2024-12-02T13:19:34,295 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87, entries=150, sequenceid=341, filesize=12.0 K 2024-12-02T13:19:34,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/ec91e3e6095b4d5492d7753c42c58450 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450 2024-12-02T13:19:34,297 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450, entries=150, sequenceid=341, filesize=12.0 K 2024-12-02T13:19:34,298 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a6dac54182c9e8f0ab33f471f7aac1b1 in 848ms, sequenceid=341, compaction requested=true 2024-12-02T13:19:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-12-02T13:19:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-12-02T13:19:34,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-02T13:19:34,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-12-02T13:19:34,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 1.0060 sec 2024-12-02T13:19:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T13:19:34,434 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:34,434 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-12-02T13:19:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:34,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:34,436 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:34,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:34,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-12-02T13:19:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:34,588 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:34,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:34,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2666776d05194a60807b99162df081b0 is 50, key is test_row_0/A:col10/1733145573567/Put/seqid=0 2024-12-02T13:19:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742364_1540 (size=12301) 2024-12-02T13:19:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:34,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145634747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145634759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145634760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145634863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145634874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:34,995 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2666776d05194a60807b99162df081b0 2024-12-02T13:19:35,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a7f6f1f7608542acab277362cf087643 is 50, key is test_row_0/B:col10/1733145573567/Put/seqid=0 2024-12-02T13:19:35,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742365_1541 (size=12301) 2024-12-02T13:19:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145635073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145635073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145635094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145635384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145635384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,405 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a7f6f1f7608542acab277362cf087643 2024-12-02T13:19:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145635403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/26f172f530084378bf7fcfbe103d692a is 50, key is test_row_0/C:col10/1733145573567/Put/seqid=0 2024-12-02T13:19:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742366_1542 (size=12301) 2024-12-02T13:19:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:35,813 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/26f172f530084378bf7fcfbe103d692a 2024-12-02T13:19:35,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/2666776d05194a60807b99162df081b0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0 2024-12-02T13:19:35,818 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T13:19:35,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/a7f6f1f7608542acab277362cf087643 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643 2024-12-02T13:19:35,821 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T13:19:35,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/26f172f530084378bf7fcfbe103d692a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a 2024-12-02T13:19:35,824 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T13:19:35,825 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a6dac54182c9e8f0ab33f471f7aac1b1 in 1236ms, sequenceid=354, compaction requested=true 2024-12-02T13:19:35,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:35,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:35,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-12-02T13:19:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-12-02T13:19:35,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-02T13:19:35,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-12-02T13:19:35,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 1.3930 sec 2024-12-02T13:19:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:35,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:35,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/63c9a2a69e8f45c98755fd44184e2f64 is 50, key is test_row_0/A:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:35,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742367_1543 (size=14741) 2024-12-02T13:19:35,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145635919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145635920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145635921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:35,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145635927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:35,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58978 deadline: 1733145635993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,000 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in a6dac54182c9e8f0ab33f471f7aac1b1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145636034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145636034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145636034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145636034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145636244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145636244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145636244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145636253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/63c9a2a69e8f45c98755fd44184e2f64 2024-12-02T13:19:36,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/f392f119ad6c4b20a34e4ceab4a92e46 is 50, key is test_row_0/B:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:36,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742368_1544 (size=12301) 2024-12-02T13:19:36,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145636553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145636564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145636564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145636564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T13:19:36,584 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:36,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-12-02T13:19:36,586 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:36,586 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:36,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:36,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/f392f119ad6c4b20a34e4ceab4a92e46 2024-12-02T13:19:36,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e912ce60761c42159f7cea9814451f5b is 50, key is test_row_0/C:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742369_1545 (size=12301) 2024-12-02T13:19:36,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e912ce60761c42159f7cea9814451f5b 2024-12-02T13:19:36,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/63c9a2a69e8f45c98755fd44184e2f64 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64 2024-12-02T13:19:36,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64, entries=200, sequenceid=374, filesize=14.4 K 2024-12-02T13:19:36,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/f392f119ad6c4b20a34e4ceab4a92e46 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46 2024-12-02T13:19:36,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T13:19:36,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/e912ce60761c42159f7cea9814451f5b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b 2024-12-02T13:19:36,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b, entries=150, sequenceid=374, filesize=12.0 K 2024-12-02T13:19:36,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a6dac54182c9e8f0ab33f471f7aac1b1 in 838ms, sequenceid=374, compaction requested=true 2024-12-02T13:19:36,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:36,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:36,733 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T13:19:36,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:36,734 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T13:19:36,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:36,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:36,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:36,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79334 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:36,735 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:36,735 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b6f6d304df2a404685f464ffad73bcf5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=77.5 K 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74454 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:36,735 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:36,735 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/4c2827e38e01474c9c8614d667337752, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=72.7 K 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c2827e38e01474c9c8614d667337752, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:36,735 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f6d304df2a404685f464ffad73bcf5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fededa3c72eb4dcea84cbf933524d094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733145571557 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaea6b7087f34e66af84cdf34e327324, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733145571557 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a8955c51e145039e1085b1822b604c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733145572199 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 695d3230d8cf4a2da3a7088468d30ca7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733145572199 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 654d020f997c46d3987ca55ef6218d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733145572891 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b65cadcd8e26428db63b7cbd415edb19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733145572891 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f6f1f7608542acab277362cf087643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733145573554 2024-12-02T13:19:36,736 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2666776d05194a60807b99162df081b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733145573554 2024-12-02T13:19:36,737 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f392f119ad6c4b20a34e4ceab4a92e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574758 2024-12-02T13:19:36,737 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63c9a2a69e8f45c98755fd44184e2f64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574746 2024-12-02T13:19:36,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:36,739 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cd66d92de3c34d3da602f9cbfc801a09 is 50, key is test_row_0/A:col10/1733145575916/Put/seqid=0 2024-12-02T13:19:36,745 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:36,746 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/6ef2d0f5503b4593bf61059c79895f73 is 50, key is test_row_0/B:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:36,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742370_1546 (size=12301) 2024-12-02T13:19:36,753 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:36,754 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/8c23023d49584cdeb3d396cb93fae7b5 is 50, key is test_row_0/A:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:36,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742371_1547 (size=13153) 2024-12-02T13:19:36,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742372_1548 (size=13153) 2024-12-02T13:19:36,759 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/6ef2d0f5503b4593bf61059c79895f73 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/6ef2d0f5503b4593bf61059c79895f73 2024-12-02T13:19:36,762 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into 6ef2d0f5503b4593bf61059c79895f73(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:36,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:36,762 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=10, startTime=1733145576733; duration=0sec 2024-12-02T13:19:36,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:36,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:36,762 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-02T13:19:36,763 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74454 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-02T13:19:36,763 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:36,764 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:36,764 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/66617f7b6dd64e17b4e154764124aa77, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=72.7 K 2024-12-02T13:19:36,764 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 66617f7b6dd64e17b4e154764124aa77, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1733145570887 2024-12-02T13:19:36,764 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8298706c632c44f984196e68b8ce7339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733145571557 2024-12-02T13:19:36,764 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2585b6c6c09d44819e528c1a4b760dd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1733145572199 2024-12-02T13:19:36,764 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ec91e3e6095b4d5492d7753c42c58450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1733145572891 2024-12-02T13:19:36,765 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f172f530084378bf7fcfbe103d692a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733145573554 2024-12-02T13:19:36,765 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e912ce60761c42159f7cea9814451f5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574758 2024-12-02T13:19:36,776 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:36,776 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/c5170fdc995742c7ac544c3fdcd597eb is 50, key is test_row_0/C:col10/1733145575894/Put/seqid=0 2024-12-02T13:19:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742373_1549 (size=13153) 2024-12-02T13:19:36,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:19:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:37,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. as already flushing 2024-12-02T13:19:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145637098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145637102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145637104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145637104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,149 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cd66d92de3c34d3da602f9cbfc801a09 2024-12-02T13:19:37,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/d1a9edef992644809249d61dc0ee1b49 is 50, key is test_row_0/B:col10/1733145575916/Put/seqid=0 2024-12-02T13:19:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742374_1550 (size=12301) 2024-12-02T13:19:37,160 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/8c23023d49584cdeb3d396cb93fae7b5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8c23023d49584cdeb3d396cb93fae7b5 2024-12-02T13:19:37,164 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 8c23023d49584cdeb3d396cb93fae7b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:37,164 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:37,164 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=10, startTime=1733145576733; duration=0sec 2024-12-02T13:19:37,164 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:37,164 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:37,182 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/c5170fdc995742c7ac544c3fdcd597eb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c5170fdc995742c7ac544c3fdcd597eb 2024-12-02T13:19:37,185 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into c5170fdc995742c7ac544c3fdcd597eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:37,185 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:37,185 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=10, startTime=1733145576734; duration=0sec 2024-12-02T13:19:37,185 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:37,186 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:37,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145637224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145637423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145637434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145637434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145637444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,558 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/d1a9edef992644809249d61dc0ee1b49 2024-12-02T13:19:37,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/72c17e2cab4142ac99f7a077a96b4f8d is 50, key is test_row_0/C:col10/1733145575916/Put/seqid=0 2024-12-02T13:19:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742375_1551 (size=12301) 2024-12-02T13:19:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59014 deadline: 1733145637733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58998 deadline: 1733145637743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145637744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145637754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:37,966 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/72c17e2cab4142ac99f7a077a96b4f8d 2024-12-02T13:19:37,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/cd66d92de3c34d3da602f9cbfc801a09 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09 2024-12-02T13:19:37,972 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T13:19:37,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/d1a9edef992644809249d61dc0ee1b49 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49 2024-12-02T13:19:37,975 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T13:19:37,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/72c17e2cab4142ac99f7a077a96b4f8d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d 2024-12-02T13:19:37,979 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d, entries=150, sequenceid=386, filesize=12.0 K 2024-12-02T13:19:37,980 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a6dac54182c9e8f0ab33f471f7aac1b1 in 1241ms, sequenceid=386, compaction requested=false 2024-12-02T13:19:37,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:37,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:37,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-02T13:19:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-12-02T13:19:37,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-02T13:19:37,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-12-02T13:19:37,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 1.3980 sec 2024-12-02T13:19:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145577096, max timestamp=1733145578249Num uniques 859; 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145577096, max timestamp=1733145578249Num uniques 859; 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145577096, max timestamp=1733145578249Num uniques 859; 2024-12-02T13:19:38,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-02T13:19:38,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:38,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:38,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0b5a774908a84fef9396e2cf7a8b8de4 is 50, key is test_row_0/A:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:38,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742376_1552 (size=14741) 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145578275, max timestamp=1733145578283Num uniques 859; 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145578275, max timestamp=1733145578283Num uniques 859; 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145578275, max timestamp=1733145578283Num uniques 859; 2024-12-02T13:19:38,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:38,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145578296, max timestamp=1733145578324Num uniques 1118; 2024-12-02T13:19:38,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T13:19:38,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145578296, max timestamp=1733145578324Num uniques 1118; 2024-12-02T13:19:38,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145578296, max timestamp=1733145578324Num uniques 1118; 2024-12-02T13:19:38,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T13:19:38,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-12-02T13:19:38,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145638359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:38,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733145578348, max timestamp=1733145578360Num uniques 1177; 2024-12-02T13:19:38,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-12-02T13:19:38,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733145578348, max timestamp=1733145578360Num uniques 1177; 2024-12-02T13:19:38,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733145578348, max timestamp=1733145578360Num uniques 1177; 2024-12-02T13:19:38,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-12-02T13:19:38,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-12-02T13:19:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145578396, max timestamp=1733145578402Num uniques 1186; 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145578396, max timestamp=1733145578402Num uniques 1186; 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733145578396, max timestamp=1733145578402Num uniques 1186; 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T13:19:38,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T13:19:38,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145578421, max timestamp=1733145578441Num uniques 1045; 2024-12-02T13:19:38,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T13:19:38,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145578421, max timestamp=1733145578441Num uniques 1045; 2024-12-02T13:19:38,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T13:19:38,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733145578421, max timestamp=1733145578441Num uniques 1045; 2024-12-02T13:19:38,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-02T13:19:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58968 deadline: 1733145638487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:38,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733145578469, max timestamp=1733145578489Num uniques 1154; 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733145578469, max timestamp=1733145578489Num uniques 1154; 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1733145578469, max timestamp=1733145578489Num uniques 1154; 2024-12-02T13:19:38,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=13920, suffix heap size=147984,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=304 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733145578534, max timestamp=1733145578551Num uniques 1163; 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733145578534, max timestamp=1733145578551Num uniques 1163; 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733145578534, max timestamp=1733145578551Num uniques 1163; 2024-12-02T13:19:38,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53258, new segment data size=12038, suffix heap size=148920,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1163, new segment cells count=263 2024-12-02T13:19:38,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58946 deadline: 1733145638595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 , the old value is region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., hostname=06e3e18f2515,45693,1733145428280, seqNum=2 because the exception is null or not the one we care about 2024-12-02T13:19:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733145578584, max timestamp=1733145578602Num uniques 1122; 2024-12-02T13:19:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T13:19:38,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733145578584, max timestamp=1733145578602Num uniques 1122; 2024-12-02T13:19:38,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T13:19:38,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733145578584, max timestamp=1733145578602Num uniques 1122; 2024-12-02T13:19:38,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=10162, suffix heap size=144656,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=222 2024-12-02T13:19:38,623 INFO [Thread-2110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2110. 2024-12-02T13:19:38,624 DEBUG [Thread-2110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,624 DEBUG [Thread-2110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,624 DEBUG [Thread-2110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,625 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-12-02T13:19:38,625 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,625 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,625 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,625 INFO [Thread-2118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2118. 2024-12-02T13:19:38,625 DEBUG [Thread-2118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,625 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,625 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,626 INFO [Thread-2099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2099. 2024-12-02T13:19:38,626 DEBUG [Thread-2099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,626 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,626 DEBUG [Thread-2099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,628 INFO [Thread-2114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2114. 2024-12-02T13:19:38,628 DEBUG [Thread-2114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,628 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,628 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-12-02T13:19:38,628 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,628 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,628 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,628 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,631 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-12-02T13:19:38,631 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,631 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,631 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,634 INFO [Thread-2112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2112. 2024-12-02T13:19:38,634 DEBUG [Thread-2112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,634 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,635 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0b5a774908a84fef9396e2cf7a8b8de4 2024-12-02T13:19:38,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ab34f1eca2a64dfab24ea31bb4eee5f8 is 50, key is test_row_0/B:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:38,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742377_1553 (size=12301) 2024-12-02T13:19:38,715 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-12-02T13:19:38,715 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:38,715 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,715 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:38,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:38,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T13:19:38,735 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:39,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ab34f1eca2a64dfab24ea31bb4eee5f8 2024-12-02T13:19:39,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/9efd8ba3ea8e46d89e22a55e9dd9c355 is 50, key is test_row_0/C:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:39,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742378_1554 (size=12301) 2024-12-02T13:19:39,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/9efd8ba3ea8e46d89e22a55e9dd9c355 2024-12-02T13:19:39,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/0b5a774908a84fef9396e2cf7a8b8de4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4 2024-12-02T13:19:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4, entries=200, sequenceid=410, filesize=14.4 K 2024-12-02T13:19:39,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ab34f1eca2a64dfab24ea31bb4eee5f8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8 2024-12-02T13:19:39,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8, entries=150, sequenceid=410, filesize=12.0 K 2024-12-02T13:19:39,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/9efd8ba3ea8e46d89e22a55e9dd9c355 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355 2024-12-02T13:19:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355, entries=150, sequenceid=410, filesize=12.0 K 2024-12-02T13:19:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=107.34 KB/109920 for a6dac54182c9e8f0ab33f471f7aac1b1 in 1250ms, sequenceid=410, compaction requested=true 2024-12-02T13:19:39,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a6dac54182c9e8f0ab33f471f7aac1b1:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/B is initiating minor compaction (all files) 2024-12-02T13:19:39,503 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/A is initiating minor compaction (all files) 2024-12-02T13:19:39,503 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/A in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:39,503 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/B in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:39,504 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8c23023d49584cdeb3d396cb93fae7b5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=39.3 K 2024-12-02T13:19:39,504 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/6ef2d0f5503b4593bf61059c79895f73, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.9 K 2024-12-02T13:19:39,504 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef2d0f5503b4593bf61059c79895f73, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574758 2024-12-02T13:19:39,504 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c23023d49584cdeb3d396cb93fae7b5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574758 2024-12-02T13:19:39,504 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd66d92de3c34d3da602f9cbfc801a09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733145575916 2024-12-02T13:19:39,504 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d1a9edef992644809249d61dc0ee1b49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733145575916 2024-12-02T13:19:39,505 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5a774908a84fef9396e2cf7a8b8de4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733145577096 2024-12-02T13:19:39,505 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ab34f1eca2a64dfab24ea31bb4eee5f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733145577096 2024-12-02T13:19:39,510 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#A#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:39,510 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:39,511 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/8679c141f1b84c628ac12b198b6a9bc3 is 50, key is test_row_0/A:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:39,511 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ffc20859b31b4a7887c2bb257df1c4cd is 50, key is test_row_0/B:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742379_1555 (size=13255) 2024-12-02T13:19:39,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742380_1556 (size=13255) 2024-12-02T13:19:39,919 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/ffc20859b31b4a7887c2bb257df1c4cd as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffc20859b31b4a7887c2bb257df1c4cd 2024-12-02T13:19:39,924 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/B of a6dac54182c9e8f0ab33f471f7aac1b1 into ffc20859b31b4a7887c2bb257df1c4cd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:39,924 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:39,924 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/B, priority=13, startTime=1733145579503; duration=0sec 2024-12-02T13:19:39,925 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:39,925 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:B 2024-12-02T13:19:39,925 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:39,925 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:39,926 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): a6dac54182c9e8f0ab33f471f7aac1b1/C is initiating minor compaction (all files) 2024-12-02T13:19:39,926 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a6dac54182c9e8f0ab33f471f7aac1b1/C in TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:39,926 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c5170fdc995742c7ac544c3fdcd597eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp, totalSize=36.9 K 2024-12-02T13:19:39,926 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c5170fdc995742c7ac544c3fdcd597eb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733145574758 2024-12-02T13:19:39,926 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c17e2cab4142ac99f7a077a96b4f8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733145575916 2024-12-02T13:19:39,926 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9efd8ba3ea8e46d89e22a55e9dd9c355, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733145577096 2024-12-02T13:19:39,928 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/8679c141f1b84c628ac12b198b6a9bc3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8679c141f1b84c628ac12b198b6a9bc3 2024-12-02T13:19:39,931 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/A of a6dac54182c9e8f0ab33f471f7aac1b1 into 8679c141f1b84c628ac12b198b6a9bc3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:39,931 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:39,931 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/A, priority=13, startTime=1733145579502; duration=0sec 2024-12-02T13:19:39,932 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:39,932 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:A 2024-12-02T13:19:39,932 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a6dac54182c9e8f0ab33f471f7aac1b1#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:39,932 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0a3b7e14154549478d344e6006d45a2b is 50, key is test_row_0/C:col10/1733145577096/Put/seqid=0 2024-12-02T13:19:39,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742381_1557 (size=13255) 2024-12-02T13:19:40,350 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/0a3b7e14154549478d344e6006d45a2b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0a3b7e14154549478d344e6006d45a2b 2024-12-02T13:19:40,356 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a6dac54182c9e8f0ab33f471f7aac1b1/C of a6dac54182c9e8f0ab33f471f7aac1b1 into 0a3b7e14154549478d344e6006d45a2b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:40,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:40,356 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1., storeName=a6dac54182c9e8f0ab33f471f7aac1b1/C, priority=13, startTime=1733145579503; duration=0sec 2024-12-02T13:19:40,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:40,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a6dac54182c9e8f0ab33f471f7aac1b1:C 2024-12-02T13:19:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:46,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a6dac54182c9e8f0ab33f471f7aac1b1 3/3 column families, dataSize=114.05 KB heapSize=319.31 KB 2024-12-02T13:19:46,045 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-12-02T13:19:46,045 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:19:46,045 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=A 2024-12-02T13:19:46,045 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:46,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=B 2024-12-02T13:19:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:19:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a6dac54182c9e8f0ab33f471f7aac1b1, store=C 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-12-02T13:19:46,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1516 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4536 rows 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1507 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4521 rows 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1508 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4518 rows 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1502 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4506 rows 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-12-02T13:19:46,046 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4527 rows 2024-12-02T13:19:46,046 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:19:46,046 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:19:46,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:46,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:46,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:19:46,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:19:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T13:19:46,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/f72d7ef83c2a4bf7ae26690444724fd0 is 50, key is test_row_0/A:col10/1733145578625/Put/seqid=0 2024-12-02T13:19:46,049 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145586049"}]},"ts":"1733145586049"} 2024-12-02T13:19:46,051 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:19:46,051 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:19:46,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:19:46,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742382_1558 (size=12301) 2024-12-02T13:19:46,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/f72d7ef83c2a4bf7ae26690444724fd0 2024-12-02T13:19:46,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, UNASSIGN}] 2024-12-02T13:19:46,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, UNASSIGN 2024-12-02T13:19:46,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a6dac54182c9e8f0ab33f471f7aac1b1, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:46,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, UNASSIGN because future has completed 2024-12-02T13:19:46,057 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:19:46,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:46,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/96bbd5122b1f494e824698c1e269ba9b is 50, key is test_row_0/B:col10/1733145578625/Put/seqid=0 2024-12-02T13:19:46,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742383_1559 (size=12301) 2024-12-02T13:19:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T13:19:46,209 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:46,209 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:19:46,209 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing a6dac54182c9e8f0ab33f471f7aac1b1, disabling compactions & flushes 2024-12-02T13:19:46,210 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T13:19:46,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/96bbd5122b1f494e824698c1e269ba9b 2024-12-02T13:19:46,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 is 50, key is test_row_0/C:col10/1733145578625/Put/seqid=0 2024-12-02T13:19:46,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742384_1560 (size=12301) 2024-12-02T13:19:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T13:19:46,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 2024-12-02T13:19:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/A/f72d7ef83c2a4bf7ae26690444724fd0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/f72d7ef83c2a4bf7ae26690444724fd0 2024-12-02T13:19:46,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/f72d7ef83c2a4bf7ae26690444724fd0, entries=150, sequenceid=566, filesize=12.0 K 2024-12-02T13:19:46,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/B/96bbd5122b1f494e824698c1e269ba9b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/96bbd5122b1f494e824698c1e269ba9b 2024-12-02T13:19:46,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/96bbd5122b1f494e824698c1e269ba9b, entries=150, sequenceid=566, filesize=12.0 K 2024-12-02T13:19:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/.tmp/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 2024-12-02T13:19:46,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758, entries=150, sequenceid=566, filesize=12.0 K 2024-12-02T13:19:46,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~319.27 KB/326928, currentSize=0 B/0 for a6dac54182c9e8f0ab33f471f7aac1b1 in 838ms, sequenceid=566, compaction requested=false 2024-12-02T13:19:46,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a6dac54182c9e8f0ab33f471f7aac1b1: 2024-12-02T13:19:46,883 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:46,883 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:46,883 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. after waiting 0 ms 2024-12-02T13:19:46,883 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:46,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/703c0370b9f24563bbac639c4d427691, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/a702d16921a64bf9b2b94e7684038405, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/93cd65b1c5cb4dffa966f8dc9ada56b6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/4df46a53434e46dd8a5bf9cc6840a1fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/73db88cf5f664ab2b62ede4bf30387c1, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b6f6d304df2a404685f464ffad73bcf5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8c23023d49584cdeb3d396cb93fae7b5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4] to archive 2024-12-02T13:19:46,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:46,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2eb34dd8fc034d09924e3b756ddc92e3 2024-12-02T13:19:46,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/189f8a81600b491abeb707d7f50899ea 2024-12-02T13:19:46,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/7143ecde6ffa4ae6b9cd28a6fc69b725 2024-12-02T13:19:46,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/17c76f3031de4f77863cf1cd0ae7f9a6 2024-12-02T13:19:46,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/703c0370b9f24563bbac639c4d427691 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/703c0370b9f24563bbac639c4d427691 2024-12-02T13:19:46,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/3e0ef0d2beb543f4aa7b5fa2bedb3334 2024-12-02T13:19:46,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/430627a1fa7d4cbf84cccc89e1e78712 2024-12-02T13:19:46,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/a702d16921a64bf9b2b94e7684038405 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/a702d16921a64bf9b2b94e7684038405 2024-12-02T13:19:46,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/6d08648db6eb4759a39c832ebffd8f0a 2024-12-02T13:19:46,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0abf3cce846043b18660191291ff8e4d 2024-12-02T13:19:46,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/93cd65b1c5cb4dffa966f8dc9ada56b6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/93cd65b1c5cb4dffa966f8dc9ada56b6 2024-12-02T13:19:46,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/ddcf4bc74c1e48e3a9baaa67b86f03bf 2024-12-02T13:19:46,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/4df46a53434e46dd8a5bf9cc6840a1fd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/4df46a53434e46dd8a5bf9cc6840a1fd 2024-12-02T13:19:46,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d49c583c42844e088e1eeb369244dbb0 2024-12-02T13:19:46,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b929b7590d474d26b704c420ca0b439b 2024-12-02T13:19:46,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/73db88cf5f664ab2b62ede4bf30387c1 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/73db88cf5f664ab2b62ede4bf30387c1 2024-12-02T13:19:46,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/5541791cde40466d92ebfdfbb4db2841 2024-12-02T13:19:46,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/d2faa9f780d74297b7d4a223501a4e30 2024-12-02T13:19:46,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b6f6d304df2a404685f464ffad73bcf5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b6f6d304df2a404685f464ffad73bcf5 2024-12-02T13:19:46,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cb1392c72ce04cf4b367b092eb7b9772 2024-12-02T13:19:46,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/aaea6b7087f34e66af84cdf34e327324 2024-12-02T13:19:46,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/695d3230d8cf4a2da3a7088468d30ca7 2024-12-02T13:19:46,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/b65cadcd8e26428db63b7cbd415edb19 2024-12-02T13:19:46,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/2666776d05194a60807b99162df081b0 2024-12-02T13:19:46,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/63c9a2a69e8f45c98755fd44184e2f64 2024-12-02T13:19:46,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8c23023d49584cdeb3d396cb93fae7b5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8c23023d49584cdeb3d396cb93fae7b5 2024-12-02T13:19:46,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/cd66d92de3c34d3da602f9cbfc801a09 2024-12-02T13:19:46,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/0b5a774908a84fef9396e2cf7a8b8de4 2024-12-02T13:19:46,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ece55b8455a341a5bf89f71b732e7339, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/7441317c1da54e3897220ae5af673f1d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/768bd00ab15e4089a2e7068a894fa2a5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/aaa5f489b06642bca2050b55d285a866, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/0835f1a050af4ba5b9b6e62df2b70aa2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/4c2827e38e01474c9c8614d667337752, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/6ef2d0f5503b4593bf61059c79895f73, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8] to archive 2024-12-02T13:19:46,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:46,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/83a8690292af433fb1f0d2fe15a17244 2024-12-02T13:19:46,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffcd2d4a824747b0b81ec1608f89f721 2024-12-02T13:19:46,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/51d42fdb0721490ea00d53cf4aef556d 2024-12-02T13:19:46,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ece55b8455a341a5bf89f71b732e7339 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ece55b8455a341a5bf89f71b732e7339 2024-12-02T13:19:46,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/1910caf021f54183a06a060d0116f08c 2024-12-02T13:19:46,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a2297995d3bc40a99aec8b171f7c35b3 2024-12-02T13:19:46,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/7441317c1da54e3897220ae5af673f1d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/7441317c1da54e3897220ae5af673f1d 2024-12-02T13:19:46,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/72bb52e79afe4aa591f51841cc78a342 2024-12-02T13:19:46,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/cdd491d696bd4398ab91c14e783026df 2024-12-02T13:19:46,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/768bd00ab15e4089a2e7068a894fa2a5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/768bd00ab15e4089a2e7068a894fa2a5 2024-12-02T13:19:46,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/c6c4f36cab394c29a60b64ac108a9a58 2024-12-02T13:19:46,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e7fdede82a2f4257993070d1cdc6ec3b 2024-12-02T13:19:46,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/aaa5f489b06642bca2050b55d285a866 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/aaa5f489b06642bca2050b55d285a866 2024-12-02T13:19:46,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a80f2c73e0f94dc69d8ef4fac0475148 2024-12-02T13:19:46,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ba202c8dfa3a488588919115f94f4d81 2024-12-02T13:19:46,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/0835f1a050af4ba5b9b6e62df2b70aa2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/0835f1a050af4ba5b9b6e62df2b70aa2 2024-12-02T13:19:46,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/880bf9264b274e3b8887a88bdd4d2bd5 2024-12-02T13:19:46,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/3b29d0897b1f445aa822a661607a0013 2024-12-02T13:19:46,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/4c2827e38e01474c9c8614d667337752 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/4c2827e38e01474c9c8614d667337752 2024-12-02T13:19:46,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/8f99964a20034d6299b5657842375f6e 2024-12-02T13:19:46,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/fededa3c72eb4dcea84cbf933524d094 2024-12-02T13:19:46,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/e5a8955c51e145039e1085b1822b604c 2024-12-02T13:19:46,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/654d020f997c46d3987ca55ef6218d87 2024-12-02T13:19:46,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/a7f6f1f7608542acab277362cf087643 2024-12-02T13:19:46,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/6ef2d0f5503b4593bf61059c79895f73 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/6ef2d0f5503b4593bf61059c79895f73 2024-12-02T13:19:46,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/f392f119ad6c4b20a34e4ceab4a92e46 2024-12-02T13:19:46,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/d1a9edef992644809249d61dc0ee1b49 2024-12-02T13:19:46,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ab34f1eca2a64dfab24ea31bb4eee5f8 2024-12-02T13:19:46,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0eb211053ac24007aaf25048b0ee749e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c13f85eebef34417b2397a5dcb42af61, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6a480e87f7bd4634b1a9cb3fa4c2c318, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6742ede8d9094fb1aeec092ef9e2f153, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/dfbb4790b6a9436cbf1c9c457d1f41bc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/66617f7b6dd64e17b4e154764124aa77, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c5170fdc995742c7ac544c3fdcd597eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355] to archive 2024-12-02T13:19:46,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:19:46,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e2885e920c8246a9a28fac9864ff61fd 2024-12-02T13:19:46,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec6901d23bca4a63a3ab389dbaedef1b 2024-12-02T13:19:46,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1f0e1ce5957f4816ac47f01143fa6f5d 2024-12-02T13:19:46,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0eb211053ac24007aaf25048b0ee749e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0eb211053ac24007aaf25048b0ee749e 2024-12-02T13:19:46,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0bd2c5bff26b4794b50aaa79666a6677 2024-12-02T13:19:46,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/eff62e8145d444f7890d4b0721df3f60 2024-12-02T13:19:46,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c13f85eebef34417b2397a5dcb42af61 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c13f85eebef34417b2397a5dcb42af61 2024-12-02T13:19:46,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ce2efba894864ab19e2cba29a3caa4fa 2024-12-02T13:19:46,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/29a3a1a0b9044f63b53a15cd7aba02bb 2024-12-02T13:19:46,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6a480e87f7bd4634b1a9cb3fa4c2c318 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6a480e87f7bd4634b1a9cb3fa4c2c318 2024-12-02T13:19:46,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/336ad6355b6942f3b9969a4e58ccbd8f 2024-12-02T13:19:46,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8cabbb7431dc4f71b18a34a721dac1af 2024-12-02T13:19:46,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6742ede8d9094fb1aeec092ef9e2f153 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/6742ede8d9094fb1aeec092ef9e2f153 2024-12-02T13:19:46,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/388c3af82f1f4471a1d919bba6070486 2024-12-02T13:19:46,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/447372a819bb4b9c84807e023ad5bf0e 2024-12-02T13:19:46,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/dfbb4790b6a9436cbf1c9c457d1f41bc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/dfbb4790b6a9436cbf1c9c457d1f41bc 2024-12-02T13:19:46,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/1df6c074de1b4617a178648524049d6a 2024-12-02T13:19:46,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/a7c0eefe378443bb9812939b81c78ba3 2024-12-02T13:19:46,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/66617f7b6dd64e17b4e154764124aa77 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/66617f7b6dd64e17b4e154764124aa77 2024-12-02T13:19:46,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/41b9f400d26f4807a3e98954a1c45ff8 2024-12-02T13:19:46,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8298706c632c44f984196e68b8ce7339 2024-12-02T13:19:46,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/2585b6c6c09d44819e528c1a4b760dd6 2024-12-02T13:19:46,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/ec91e3e6095b4d5492d7753c42c58450 2024-12-02T13:19:46,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/26f172f530084378bf7fcfbe103d692a 2024-12-02T13:19:46,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c5170fdc995742c7ac544c3fdcd597eb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/c5170fdc995742c7ac544c3fdcd597eb 2024-12-02T13:19:46,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/e912ce60761c42159f7cea9814451f5b 2024-12-02T13:19:46,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/72c17e2cab4142ac99f7a077a96b4f8d 2024-12-02T13:19:46,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/9efd8ba3ea8e46d89e22a55e9dd9c355 2024-12-02T13:19:46,966 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/recovered.edits/569.seqid, newMaxSeqId=569, maxSeqId=1 2024-12-02T13:19:46,967 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1. 2024-12-02T13:19:46,967 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for a6dac54182c9e8f0ab33f471f7aac1b1: Waiting for close lock at 1733145586209Running coprocessor pre-close hooks at 1733145586209Disabling compacts and flushes for region at 1733145586209Disabling writes for close at 1733145586883 (+674 ms)Writing region close event to WAL at 1733145586964 (+81 ms)Running coprocessor post-close hooks at 1733145586967 (+3 ms)Closed at 1733145586967 2024-12-02T13:19:46,969 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:46,969 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=a6dac54182c9e8f0ab33f471f7aac1b1, regionState=CLOSED 2024-12-02T13:19:46,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:46,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-12-02T13:19:46,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure a6dac54182c9e8f0ab33f471f7aac1b1, server=06e3e18f2515,45693,1733145428280 in 915 msec 2024-12-02T13:19:46,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T13:19:46,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a6dac54182c9e8f0ab33f471f7aac1b1, UNASSIGN in 920 msec 2024-12-02T13:19:46,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-02T13:19:46,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 923 msec 2024-12-02T13:19:46,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145586978"}]},"ts":"1733145586978"} 2024-12-02T13:19:46,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:19:46,980 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:19:46,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 933 msec 2024-12-02T13:19:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T13:19:47,183 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:47,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:19:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,185 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-12-02T13:19:47,186 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=157, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:47,190 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/recovered.edits] 2024-12-02T13:19:47,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8679c141f1b84c628ac12b198b6a9bc3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/8679c141f1b84c628ac12b198b6a9bc3 2024-12-02T13:19:47,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/f72d7ef83c2a4bf7ae26690444724fd0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/A/f72d7ef83c2a4bf7ae26690444724fd0 2024-12-02T13:19:47,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/96bbd5122b1f494e824698c1e269ba9b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/96bbd5122b1f494e824698c1e269ba9b 2024-12-02T13:19:47,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffc20859b31b4a7887c2bb257df1c4cd to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/B/ffc20859b31b4a7887c2bb257df1c4cd 2024-12-02T13:19:47,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0a3b7e14154549478d344e6006d45a2b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/0a3b7e14154549478d344e6006d45a2b 2024-12-02T13:19:47,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/C/8ba7ad8fa7bf4bd59ee357bcf6ecc758 2024-12-02T13:19:47,198 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/recovered.edits/569.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1/recovered.edits/569.seqid 2024-12-02T13:19:47,198 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/a6dac54182c9e8f0ab33f471f7aac1b1 2024-12-02T13:19:47,198 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:19:47,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=157, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,201 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:19:47,203 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:19:47,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=157, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,204 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:19:47,205 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145587204"}]},"ts":"9223372036854775807"} 2024-12-02T13:19:47,207 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:19:47,207 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a6dac54182c9e8f0ab33f471f7aac1b1, NAME => 'TestAcidGuarantees,,1733145556411.a6dac54182c9e8f0ab33f471f7aac1b1.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:19:47,207 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:19:47,207 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145587207"}]},"ts":"9223372036854775807"} 2024-12-02T13:19:47,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:19:47,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=157, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-12-02T13:19:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-12-02T13:19:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:19:47,296 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:47,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 461), ProcessCount=11 (was 11), AvailableMemoryMB=2995 (was 2530) - AvailableMemoryMB LEAK? - 2024-12-02T13:19:47,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=2993 2024-12-02T13:19:47,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:19:47,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:19:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:47,311 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T13:19:47,312 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:47,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 158 2024-12-02T13:19:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:47,312 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T13:19:47,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742385_1561 (size=1061) 2024-12-02T13:19:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:47,718 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b 2024-12-02T13:19:47,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742386_1562 (size=53) 2024-12-02T13:19:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:48,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 454ba568ee8de3afe25b6c498761192b, disabling compactions & flushes 2024-12-02T13:19:48,124 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. after waiting 0 ms 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,124 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 454ba568ee8de3afe25b6c498761192b: Waiting for close lock at 1733145588124Disabling compacts and flushes for region at 1733145588124Disabling writes for close at 1733145588124Writing region close event to WAL at 1733145588124Closed at 1733145588124 2024-12-02T13:19:48,125 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T13:19:48,126 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733145588125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733145588125"}]},"ts":"1733145588125"} 2024-12-02T13:19:48,127 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T13:19:48,128 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T13:19:48,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145588128"}]},"ts":"1733145588128"} 2024-12-02T13:19:48,130 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T13:19:48,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, ASSIGN}] 2024-12-02T13:19:48,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, ASSIGN 2024-12-02T13:19:48,132 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, ASSIGN; state=OFFLINE, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=false 2024-12-02T13:19:48,283 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:48,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, ASSIGN because future has completed 2024-12-02T13:19:48,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:48,445 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,445 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7752): Opening region: {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:19:48,445 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,445 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:19:48,445 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7794): checking encryption for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,446 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7797): checking classloading for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,447 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,448 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:48,448 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName A 2024-12-02T13:19:48,448 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:48,448 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:48,448 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,449 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:48,449 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName B 2024-12-02T13:19:48,449 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:48,449 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:48,450 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,450 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:48,450 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName C 2024-12-02T13:19:48,450 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:48,451 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:48,451 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,451 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1038): replaying wal for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,451 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,452 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,452 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1048): stopping wal replay for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,452 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1060): Cleaning up temporary data for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,453 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:19:48,454 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1093): writing seq id for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:48,455 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T13:19:48,455 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1114): Opened 454ba568ee8de3afe25b6c498761192b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67482013, jitterRate=0.005560353398323059}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:19:48,455 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:48,456 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1006): Region open journal for 454ba568ee8de3afe25b6c498761192b: Running coprocessor pre-open hook at 1733145588446Writing region info on filesystem at 1733145588446Initializing all the Stores at 1733145588446Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145588446Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145588446Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145588446Cleaning up temporary data from old regions at 1733145588452 (+6 ms)Running coprocessor post-open hooks at 1733145588455 (+3 ms)Region opened successfully at 1733145588456 (+1 ms) 2024-12-02T13:19:48,457 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., pid=160, masterSystemTime=1733145588443 2024-12-02T13:19:48,458 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,458 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:48,458 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=OPEN, openSeqNum=2, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:48,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:48,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-02T13:19:48,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 in 173 msec 2024-12-02T13:19:48,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T13:19:48,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, ASSIGN in 332 msec 2024-12-02T13:19:48,464 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T13:19:48,465 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145588464"}]},"ts":"1733145588464"} 2024-12-02T13:19:48,466 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T13:19:48,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T13:19:48,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-12-02T13:19:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T13:19:49,476 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:49,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:49,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:49,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:49,481 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4f580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:49,483 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688de195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:49,484 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:49,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:49,488 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T13:19:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 06e3e18f2515,35237,1733145427680 2024-12-02T13:19:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d1d91a1 2024-12-02T13:19:49,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T13:19:49,489 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T13:19:49,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T13:19:49,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T13:19:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T13:19:49,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742387_1563 (size=1097) 2024-12-02T13:19:49,903 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T13:19:49,903 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T13:19:49,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:19:49,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE}] 2024-12-02T13:19:49,912 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE 2024-12-02T13:19:49,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:49,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE because future has completed 2024-12-02T13:19:49,915 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:19:49,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:50,068 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(122): Close 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,068 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:19:50,068 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1722): Closing 454ba568ee8de3afe25b6c498761192b, disabling compactions & flushes 2024-12-02T13:19:50,068 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,068 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,068 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. after waiting 0 ms 2024-12-02T13:19:50,068 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,074 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T13:19:50,075 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,076 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1676): Region close journal for 454ba568ee8de3afe25b6c498761192b: Waiting for close lock at 1733145590068Running coprocessor pre-close hooks at 1733145590068Disabling compacts and flushes for region at 1733145590068Disabling writes for close at 1733145590068Writing region close event to WAL at 1733145590069 (+1 ms)Running coprocessor post-close hooks at 1733145590075 (+6 ms)Closed at 1733145590075 2024-12-02T13:19:50,076 WARN [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionServer(3299): Not adding moved region record: 454ba568ee8de3afe25b6c498761192b to self. 2024-12-02T13:19:50,079 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(157): Closed 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,079 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=CLOSED 2024-12-02T13:19:50,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:50,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-02T13:19:50,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 in 167 msec 2024-12-02T13:19:50,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE; state=CLOSED, location=06e3e18f2515,45693,1733145428280; forceNewPlan=false, retain=true 2024-12-02T13:19:50,235 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=OPENING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE because future has completed 2024-12-02T13:19:50,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:19:50,398 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,398 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7752): Opening region: {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T13:19:50,399 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,399 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T13:19:50,399 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7794): checking encryption for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,399 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7797): checking classloading for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,401 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,402 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:50,402 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName A 2024-12-02T13:19:50,404 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:50,405 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:50,405 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,405 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:50,406 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName B 2024-12-02T13:19:50,406 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:50,406 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:50,406 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,406 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T13:19:50,406 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 454ba568ee8de3afe25b6c498761192b columnFamilyName C 2024-12-02T13:19:50,406 DEBUG [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:50,407 INFO [StoreOpener-454ba568ee8de3afe25b6c498761192b-1 {}] regionserver.HStore(327): Store=454ba568ee8de3afe25b6c498761192b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T13:19:50,407 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,407 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1038): replaying wal for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,407 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,408 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,409 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1048): stopping wal replay for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,409 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1060): Cleaning up temporary data for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,409 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T13:19:50,410 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1093): writing seq id for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,410 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1114): Opened 454ba568ee8de3afe25b6c498761192b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60451182, jitterRate=-0.09920719265937805}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T13:19:50,410 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,410 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1006): Region open journal for 454ba568ee8de3afe25b6c498761192b: Running coprocessor pre-open hook at 1733145590399Writing region info on filesystem at 1733145590399Initializing all the Stores at 1733145590401 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145590401Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145590401Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733145590401Cleaning up temporary data from old regions at 1733145590409 (+8 ms)Running coprocessor post-open hooks at 1733145590410 (+1 ms)Region opened successfully at 1733145590410 2024-12-02T13:19:50,411 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., pid=165, masterSystemTime=1733145590391 2024-12-02T13:19:50,412 DEBUG [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,412 INFO [RS_OPEN_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,413 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=OPEN, openSeqNum=5, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:19:50,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=163 2024-12-02T13:19:50,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=163, state=SUCCESS, hasLock=false; OpenRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 in 176 msec 2024-12-02T13:19:50,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T13:19:50,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, REOPEN/MOVE in 505 msec 2024-12-02T13:19:50,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-02T13:19:50,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-12-02T13:19:50,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-12-02T13:19:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-02T13:19:50,422 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:50,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c29eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667222d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,426 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d0ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee7f095, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f55c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfc7843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a1476d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,433 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0c8676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbdbc1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,436 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc3c789, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af73db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf9d6f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2103660d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb6d812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,442 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379ad8c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57209fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,445 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a96f266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,446 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec9054a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 06e3e18f2515,35237,-1 for getting cluster id 2024-12-02T13:19:50,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T13:19:50,448 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6274f614-26a8-4bfb-bcd2-91fd56d07722' 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6274f614-26a8-4bfb-bcd2-91fd56d07722" 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ff26ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [06e3e18f2515,35237,-1] 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T13:19:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:19:50,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T13:19:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2345b152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T13:19:50,450 DEBUG [Thread-2518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,450 DEBUG [Thread-2520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,450 DEBUG [Thread-2511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,450 DEBUG [Thread-2509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,450 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,450 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:50,451 DEBUG [Thread-2513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,452 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-02T13:19:50,453 DEBUG [Thread-2516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,454 DEBUG [Thread-2522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T13:19:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:50,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,454 DEBUG [Thread-2516 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=06e3e18f2515,45693,1733145428280, seqNum=-1] 2024-12-02T13:19:50,455 DEBUG [Thread-2516 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T13:19:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,455 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:50,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T13:19:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,456 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:50,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,461 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5] 2024-12-02T13:19:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145650477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020abfcfd95c674791a1911aa84ca1e247_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145590461/Put/seqid=0 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145590473, max timestamp=1733145590479Num uniques 859; 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145590473, max timestamp=1733145590479Num uniques 859; 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733145590473, max timestamp=1733145590479Num uniques 859; 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:50,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T13:19:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742388_1564 (size=17034) 2024-12-02T13:19:50,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:50,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020abfcfd95c674791a1911aa84ca1e247_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020abfcfd95c674791a1911aa84ca1e247_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:50,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d26b01171f36490b8adc84f0860dd7f7, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:50,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d26b01171f36490b8adc84f0860dd7f7 is 175, key is test_row_0/A:col10/1733145590461/Put/seqid=0 2024-12-02T13:19:50,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742389_1565 (size=48139) 2024-12-02T13:19:50,495 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d26b01171f36490b8adc84f0860dd7f7 2024-12-02T13:19:50,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145650494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145650494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145650496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145650497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/56fb9d6dab1f483fbab6330ec399cf14 is 50, key is test_row_0/B:col10/1733145590461/Put/seqid=0 2024-12-02T13:19:50,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742390_1566 (size=12001) 2024-12-02T13:19:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145650584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145650604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145650604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145650604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:50,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145650609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145650794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145650814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145650814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145650814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145650814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:50,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:50,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/56fb9d6dab1f483fbab6330ec399cf14 2024-12-02T13:19:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:50,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:50,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f5be943de4f04258b86f2a8e4a356b35 is 50, key is test_row_0/C:col10/1733145590461/Put/seqid=0 2024-12-02T13:19:50,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742391_1567 (size=12001) 2024-12-02T13:19:51,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:51,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:51,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:51,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:51,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:51,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145651104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145651124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145651124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145651124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145651124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:51,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:51,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f5be943de4f04258b86f2a8e4a356b35 2024-12-02T13:19:51,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d26b01171f36490b8adc84f0860dd7f7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7 2024-12-02T13:19:51,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7, entries=250, sequenceid=15, filesize=47.0 K 2024-12-02T13:19:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/56fb9d6dab1f483fbab6330ec399cf14 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14 2024-12-02T13:19:51,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T13:19:51,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f5be943de4f04258b86f2a8e4a356b35 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35 2024-12-02T13:19:51,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T13:19:51,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 454ba568ee8de3afe25b6c498761192b in 893ms, sequenceid=15, compaction requested=false 2024-12-02T13:19:51,355 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T13:19:51,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:51,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-02T13:19:51,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:51,373 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:51,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029e626db340824a37ba2db4c7da41ed97_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145590496/Put/seqid=0 2024-12-02T13:19:51,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742392_1568 (size=12154) 2024-12-02T13:19:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:51,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:51,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145651635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145651634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145651635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145651638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145651639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145651744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145651744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145651755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:51,794 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029e626db340824a37ba2db4c7da41ed97_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029e626db340824a37ba2db4c7da41ed97_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:51,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/928c329bcdce4374aefa351734093f18, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:51,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/928c329bcdce4374aefa351734093f18 is 175, key is test_row_0/A:col10/1733145590496/Put/seqid=0 2024-12-02T13:19:51,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742393_1569 (size=30955) 2024-12-02T13:19:51,806 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/928c329bcdce4374aefa351734093f18 2024-12-02T13:19:51,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/bb8e51cf316648a883bcfadb876e6c3e is 50, key is test_row_0/B:col10/1733145590496/Put/seqid=0 2024-12-02T13:19:51,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742394_1570 (size=12001) 2024-12-02T13:19:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145651953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145651953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145651964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,228 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/bb8e51cf316648a883bcfadb876e6c3e 2024-12-02T13:19:52,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fba4ccdb162c4c97bd7819379953b08b is 50, key is test_row_0/C:col10/1733145590496/Put/seqid=0 2024-12-02T13:19:52,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742395_1571 (size=12001) 2024-12-02T13:19:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145652264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145652274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145652274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,483 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T13:19:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:52,636 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fba4ccdb162c4c97bd7819379953b08b 2024-12-02T13:19:52,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/928c329bcdce4374aefa351734093f18 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18 2024-12-02T13:19:52,646 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T13:19:52,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/bb8e51cf316648a883bcfadb876e6c3e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e 2024-12-02T13:19:52,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145652653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145652654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,654 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:52,654 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,657 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T13:19:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fba4ccdb162c4c97bd7819379953b08b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b 2024-12-02T13:19:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T13:19:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 454ba568ee8de3afe25b6c498761192b in 1305ms, sequenceid=49, compaction requested=false 2024-12-02T13:19:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-02T13:19:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-02T13:19:52,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-02T13:19:52,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2240 sec 2024-12-02T13:19:52,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.2330 sec 2024-12-02T13:19:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:52,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:52,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:52,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e76ed04815cc4b8e854e67e07349fc41_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:52,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742396_1572 (size=14594) 2024-12-02T13:19:52,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:52,790 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e76ed04815cc4b8e854e67e07349fc41_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e76ed04815cc4b8e854e67e07349fc41_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:52,792 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/c36c990e510e4f41a825b37d4542fec4, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/c36c990e510e4f41a825b37d4542fec4 is 175, key is test_row_0/A:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:52,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742397_1573 (size=39549) 2024-12-02T13:19:52,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/c36c990e510e4f41a825b37d4542fec4 2024-12-02T13:19:52,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/2f502b3b9b2940f6923babfb68af0de2 is 50, key is test_row_0/B:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145652803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742398_1574 (size=12001) 2024-12-02T13:19:52,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/2f502b3b9b2940f6923babfb68af0de2 2024-12-02T13:19:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145652806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145652808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c3b577583bea46148aafab9121e22940 is 50, key is test_row_0/C:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:52,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742399_1575 (size=12001) 2024-12-02T13:19:52,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c3b577583bea46148aafab9121e22940 2024-12-02T13:19:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/c36c990e510e4f41a825b37d4542fec4 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4 2024-12-02T13:19:52,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4, entries=200, sequenceid=61, filesize=38.6 K 2024-12-02T13:19:52,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/2f502b3b9b2940f6923babfb68af0de2 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2 2024-12-02T13:19:52,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T13:19:52,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c3b577583bea46148aafab9121e22940 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940 2024-12-02T13:19:52,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T13:19:52,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 454ba568ee8de3afe25b6c498761192b in 77ms, sequenceid=61, compaction requested=true 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:52,852 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:52,852 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:19:52,853 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:52,853 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:52,853 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=115.9 K 2024-12-02T13:19:52,853 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.2 K 2024-12-02T13:19:52,853 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4] 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 56fb9d6dab1f483fbab6330ec399cf14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145590458 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d26b01171f36490b8adc84f0860dd7f7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145590458 2024-12-02T13:19:52,853 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting bb8e51cf316648a883bcfadb876e6c3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145590492 2024-12-02T13:19:52,854 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928c329bcdce4374aefa351734093f18, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145590492 2024-12-02T13:19:52,854 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f502b3b9b2940f6923babfb68af0de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145591616 2024-12-02T13:19:52,854 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36c990e510e4f41a825b37d4542fec4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145591616 2024-12-02T13:19:52,858 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,860 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202dfdf3e0dd3a544af804e1b5f8ddefe0f_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,860 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:52,861 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/43e7179bf5c94a4f8a1a380bf02b2886 is 50, key is test_row_0/B:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:52,862 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202dfdf3e0dd3a544af804e1b5f8ddefe0f_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,862 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dfdf3e0dd3a544af804e1b5f8ddefe0f_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742400_1576 (size=12104) 2024-12-02T13:19:52,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742401_1577 (size=4469) 2024-12-02T13:19:52,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:52,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b35cb5cbf764997a51a61c1e977cd2e_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145592915/Put/seqid=0 2024-12-02T13:19:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742402_1578 (size=14594) 2024-12-02T13:19:52,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:52,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026b35cb5cbf764997a51a61c1e977cd2e_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b35cb5cbf764997a51a61c1e977cd2e_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/11eab79a151b423cbc89ad1d1e8182fb, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145652932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/11eab79a151b423cbc89ad1d1e8182fb is 175, key is test_row_0/A:col10/1733145592915/Put/seqid=0 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145652935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145652935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:52,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742403_1579 (size=39549) 2024-12-02T13:19:52,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/11eab79a151b423cbc89ad1d1e8182fb 2024-12-02T13:19:52,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ebde2f6380bb41bf9ca6e60d63b67512 is 50, key is test_row_0/B:col10/1733145592915/Put/seqid=0 2024-12-02T13:19:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742404_1580 (size=12001) 2024-12-02T13:19:53,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ebde2f6380bb41bf9ca6e60d63b67512 2024-12-02T13:19:53,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/a84db884e8034deebe526b86d6ee724e is 50, key is test_row_0/C:col10/1733145592915/Put/seqid=0 2024-12-02T13:19:53,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145653044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145653046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145653046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742405_1581 (size=12001) 2024-12-02T13:19:53,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/a84db884e8034deebe526b86d6ee724e 2024-12-02T13:19:53,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/11eab79a151b423cbc89ad1d1e8182fb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb 2024-12-02T13:19:53,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb, entries=200, sequenceid=82, filesize=38.6 K 2024-12-02T13:19:53,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ebde2f6380bb41bf9ca6e60d63b67512 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512 2024-12-02T13:19:53,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T13:19:53,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/a84db884e8034deebe526b86d6ee724e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e 2024-12-02T13:19:53,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T13:19:53,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 454ba568ee8de3afe25b6c498761192b in 168ms, sequenceid=82, compaction requested=true 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T13:19:53,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:53,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:53,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:53,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f39c0cdee2b44bd183a18cc54c315261_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:53,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742406_1582 (size=12154) 2024-12-02T13:19:53,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145653280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145653280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,286 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/43e7179bf5c94a4f8a1a380bf02b2886 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/43e7179bf5c94a4f8a1a380bf02b2886 2024-12-02T13:19:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145653284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,289 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 43e7179bf5c94a4f8a1a380bf02b2886(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:53,289 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:53,290 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145592852; duration=0sec 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:19:53,290 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:19:53,291 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:19:53,291 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:19:53,291 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:53,291 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=46.9 K 2024-12-02T13:19:53,291 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f5be943de4f04258b86f2a8e4a356b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733145590458 2024-12-02T13:19:53,292 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fba4ccdb162c4c97bd7819379953b08b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733145590492 2024-12-02T13:19:53,292 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b577583bea46148aafab9121e22940, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145591616 2024-12-02T13:19:53,292 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting a84db884e8034deebe526b86d6ee724e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145592804 2024-12-02T13:19:53,298 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:53,298 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f0f492e28db0415b8fc82cf6540642df is 50, key is test_row_0/C:col10/1733145592915/Put/seqid=0 2024-12-02T13:19:53,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742407_1583 (size=12139) 2024-12-02T13:19:53,311 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#488 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:53,311 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f06438a762a84801a97bfccb56042c43 is 175, key is test_row_0/A:col10/1733145591637/Put/seqid=0 2024-12-02T13:19:53,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742408_1584 (size=31058) 2024-12-02T13:19:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145653393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145653393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145653394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145653604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145653604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145653604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:53,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f39c0cdee2b44bd183a18cc54c315261_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f39c0cdee2b44bd183a18cc54c315261_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:53,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/a2dbd409b2544ea78e6493db4737fc18, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:53,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/a2dbd409b2544ea78e6493db4737fc18 is 175, key is test_row_0/A:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:53,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742409_1585 (size=30955) 2024-12-02T13:19:53,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f0f492e28db0415b8fc82cf6540642df as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0f492e28db0415b8fc82cf6540642df 2024-12-02T13:19:53,708 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into f0f492e28db0415b8fc82cf6540642df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:53,708 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=12, startTime=1733145593084; duration=0sec 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:19:53,708 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:53,709 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:19:53,717 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f06438a762a84801a97bfccb56042c43 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43 2024-12-02T13:19:53,720 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into f06438a762a84801a97bfccb56042c43(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:53,720 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:53,720 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145592852; duration=0sec 2024-12-02T13:19:53,720 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:53,720 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:19:53,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145653913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145653913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145653913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/a2dbd409b2544ea78e6493db4737fc18 2024-12-02T13:19:54,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d0265be9b03462d9e8da1d67f5b64c3 is 50, key is test_row_0/B:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:54,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742410_1586 (size=12001) 2024-12-02T13:19:54,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d0265be9b03462d9e8da1d67f5b64c3 2024-12-02T13:19:54,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f32d2f3b19e24c53b83f757d894a6dd8 is 50, key is test_row_0/C:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:54,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742411_1587 (size=12001) 2024-12-02T13:19:54,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f32d2f3b19e24c53b83f757d894a6dd8 2024-12-02T13:19:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/a2dbd409b2544ea78e6493db4737fc18 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18 2024-12-02T13:19:54,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18, entries=150, sequenceid=95, filesize=30.2 K 2024-12-02T13:19:54,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d0265be9b03462d9e8da1d67f5b64c3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3 2024-12-02T13:19:54,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T13:19:54,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f32d2f3b19e24c53b83f757d894a6dd8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8 2024-12-02T13:19:54,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T13:19:54,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 454ba568ee8de3afe25b6c498761192b in 927ms, sequenceid=95, compaction requested=true 2024-12-02T13:19:54,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:54,182 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:54,183 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:54,183 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:54,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:54,183 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:19:54,183 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,183 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=99.2 K 2024-12-02T13:19:54,183 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,183 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18] 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06438a762a84801a97bfccb56042c43, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145591616 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:19:54,184 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,184 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/43e7179bf5c94a4f8a1a380bf02b2886, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.3 K 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11eab79a151b423cbc89ad1d1e8182fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145592791 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e7179bf5c94a4f8a1a380bf02b2886, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733145591616 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2dbd409b2544ea78e6493db4737fc18, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733145592931 2024-12-02T13:19:54,184 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ebde2f6380bb41bf9ca6e60d63b67512, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145592804 2024-12-02T13:19:54,190 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0265be9b03462d9e8da1d67f5b64c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733145592931 2024-12-02T13:19:54,196 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:54,200 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:54,200 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/18c8c6e508a449a49d123246b968e395 is 50, key is test_row_0/B:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:54,206 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d0f70ccc5bf640748caafe9757d3cc64_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:54,207 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d0f70ccc5bf640748caafe9757d3cc64_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:54,208 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d0f70ccc5bf640748caafe9757d3cc64_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:54,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742413_1589 (size=4469) 2024-12-02T13:19:54,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742412_1588 (size=12207) 2024-12-02T13:19:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:54,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:54,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:54,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021f1645bc6d434798a5debec5e1e222b7_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145594425/Put/seqid=0 2024-12-02T13:19:54,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742414_1590 (size=14594) 2024-12-02T13:19:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145654438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145654437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145654439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145654554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145654554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145654554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,612 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#497 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:54,612 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/bcf9571fbcd645eba07bb6fc483c12b3 is 175, key is test_row_0/A:col10/1733145593254/Put/seqid=0 2024-12-02T13:19:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T13:19:54,614 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:54,615 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:54,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742415_1591 (size=31161) 2024-12-02T13:19:54,615 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/18c8c6e508a449a49d123246b968e395 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/18c8c6e508a449a49d123246b968e395 2024-12-02T13:19:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-02T13:19:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:54,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:54,617 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:54,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:54,620 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 18c8c6e508a449a49d123246b968e395(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:54,620 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:54,620 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145594183; duration=0sec 2024-12-02T13:19:54,620 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:54,620 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:19:54,620 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:19:54,626 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:54,626 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:54,626 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:54,626 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:19:54,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145654663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,665 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145654674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,675 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145654764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145654764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145654764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:54,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:54,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021f1645bc6d434798a5debec5e1e222b7_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021f1645bc6d434798a5debec5e1e222b7_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:54,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f6226b530f4a4f1496a15ad87939400e, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:54,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f6226b530f4a4f1496a15ad87939400e is 175, key is test_row_0/A:col10/1733145594425/Put/seqid=0 2024-12-02T13:19:54,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742416_1592 (size=39549) 2024-12-02T13:19:54,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:54,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:55,019 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/bcf9571fbcd645eba07bb6fc483c12b3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3 2024-12-02T13:19:55,022 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into bcf9571fbcd645eba07bb6fc483c12b3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:55,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:55,022 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145594182; duration=0sec 2024-12-02T13:19:55,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:55,022 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:19:55,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145655073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145655074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145655074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f6226b530f4a4f1496a15ad87939400e 2024-12-02T13:19:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:55,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1d2dbfcc0e8c4776810741b969bf6943 is 50, key is test_row_0/B:col10/1733145594425/Put/seqid=0 2024-12-02T13:19:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742417_1593 (size=12001) 2024-12-02T13:19:55,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145655584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145655583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145655584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:55,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1d2dbfcc0e8c4776810741b969bf6943 2024-12-02T13:19:55,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/2b32d0810aae466c9ba6c6e82150c4cc is 50, key is test_row_0/C:col10/1733145594425/Put/seqid=0 2024-12-02T13:19:55,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742418_1594 (size=12001) 2024-12-02T13:19:55,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:55,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:55,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:55,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:56,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/2b32d0810aae466c9ba6c6e82150c4cc 2024-12-02T13:19:56,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/f6226b530f4a4f1496a15ad87939400e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e 2024-12-02T13:19:56,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e, entries=200, sequenceid=120, filesize=38.6 K 2024-12-02T13:19:56,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1d2dbfcc0e8c4776810741b969bf6943 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943 2024-12-02T13:19:56,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943, entries=150, sequenceid=120, filesize=11.7 K 2024-12-02T13:19:56,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/2b32d0810aae466c9ba6c6e82150c4cc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc 2024-12-02T13:19:56,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc, entries=150, sequenceid=120, filesize=11.7 K 2024-12-02T13:19:56,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 454ba568ee8de3afe25b6c498761192b in 1654ms, sequenceid=120, compaction requested=true 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:56,081 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:19:56,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:19:56,081 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. because compaction request was cancelled 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:19:56,082 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:19:56,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:19:56,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:19:56,083 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:56,083 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0f492e28db0415b8fc82cf6540642df, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.3 K 2024-12-02T13:19:56,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f492e28db0415b8fc82cf6540642df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733145592804 2024-12-02T13:19:56,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f32d2f3b19e24c53b83f757d894a6dd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733145592931 2024-12-02T13:19:56,083 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b32d0810aae466c9ba6c6e82150c4cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733145594425 2024-12-02T13:19:56,088 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:19:56,088 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/ff875763c66a4d9abd077adbe6209035 is 50, key is test_row_0/C:col10/1733145594425/Put/seqid=0 2024-12-02T13:19:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742419_1595 (size=12241) 2024-12-02T13:19:56,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-02T13:19:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:56,144 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:56,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:19:56,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021385387da15e4908970e471d4b81b98a_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145594436/Put/seqid=0 2024-12-02T13:19:56,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742420_1596 (size=12204) 2024-12-02T13:19:56,495 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/ff875763c66a4d9abd077adbe6209035 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/ff875763c66a4d9abd077adbe6209035 2024-12-02T13:19:56,498 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into ff875763c66a4d9abd077adbe6209035(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:19:56,498 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:56,498 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145596081; duration=0sec 2024-12-02T13:19:56,498 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:19:56,498 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:19:56,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:56,555 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021385387da15e4908970e471d4b81b98a_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021385387da15e4908970e471d4b81b98a_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:56,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/72ae8d3292ba48fca32cbd3df3180704, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:56,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/72ae8d3292ba48fca32cbd3df3180704 is 175, key is test_row_0/A:col10/1733145594436/Put/seqid=0 2024-12-02T13:19:56,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742421_1597 (size=31005) 2024-12-02T13:19:56,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:56,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145596609, max timestamp=1733145596619Num uniques 859; 2024-12-02T13:19:56,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:56,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145596609, max timestamp=1733145596619Num uniques 859; 2024-12-02T13:19:56,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:56,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145596609, max timestamp=1733145596619Num uniques 859; 2024-12-02T13:19:56,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:19:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145656642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145656643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145656646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145656754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145656754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145656754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:56,960 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/72ae8d3292ba48fca32cbd3df3180704 2024-12-02T13:19:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/94730f5cb6804cb494e9b8963a0d8022 is 50, key is test_row_0/B:col10/1733145594436/Put/seqid=0 2024-12-02T13:19:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145656964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145656964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145656964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:56,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:56,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742422_1598 (size=12051) 2024-12-02T13:19:57,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145657273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145657274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145657274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,372 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/94730f5cb6804cb494e9b8963a0d8022 2024-12-02T13:19:57,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/41bfeae4db244572b362f2e3d13bde84 is 50, key is test_row_0/C:col10/1733145594436/Put/seqid=0 2024-12-02T13:19:57,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742423_1599 (size=12051) 2024-12-02T13:19:57,781 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/41bfeae4db244572b362f2e3d13bde84 2024-12-02T13:19:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/72ae8d3292ba48fca32cbd3df3180704 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704 2024-12-02T13:19:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145657784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145657784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:57,787 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704, entries=150, sequenceid=132, filesize=30.3 K 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145657784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:57,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/94730f5cb6804cb494e9b8963a0d8022 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022 2024-12-02T13:19:57,790 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022, entries=150, sequenceid=132, filesize=11.8 K 2024-12-02T13:19:57,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/41bfeae4db244572b362f2e3d13bde84 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84 2024-12-02T13:19:57,794 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84, entries=150, sequenceid=132, filesize=11.8 K 2024-12-02T13:19:57,794 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 454ba568ee8de3afe25b6c498761192b in 1650ms, sequenceid=132, compaction requested=true 2024-12-02T13:19:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:19:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:57,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-02T13:19:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-02T13:19:57,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-02T13:19:57,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1790 sec 2024-12-02T13:19:57,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 3.1830 sec 2024-12-02T13:19:58,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T13:19:58,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T13:19:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:58,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:19:58,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:19:58,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028328577cbfaa4442b623ae95d25c4448_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145596634/Put/seqid=0 2024-12-02T13:19:58,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742424_1600 (size=14794) 2024-12-02T13:19:58,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145658710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145658713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-02T13:19:58,784 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:19:58,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:19:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-02T13:19:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:19:58,785 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:19:58,786 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:19:58,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:19:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145658794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,796 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145658804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145658804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,807 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,807 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:19:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145658823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145658823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:58,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:19:58,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:58,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:58,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145659033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145659033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:19:59,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028328577cbfaa4442b623ae95d25c4448_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028328577cbfaa4442b623ae95d25c4448_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:19:59,109 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/427efafc099d496883e363c37237ad52, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:19:59,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/427efafc099d496883e363c37237ad52 is 175, key is test_row_0/A:col10/1733145596634/Put/seqid=0 2024-12-02T13:19:59,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742425_1601 (size=39749) 2024-12-02T13:19:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:19:59,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145659344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145659344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:19:59,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/427efafc099d496883e363c37237ad52 2024-12-02T13:19:59,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/cec3120b500945689aae21e5be9db024 is 50, key is test_row_0/B:col10/1733145596634/Put/seqid=0 2024-12-02T13:19:59,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742426_1602 (size=12151) 2024-12-02T13:19:59,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:19:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:19:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:19:59,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:19:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145659854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:19:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145659854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:19:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:19:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:19:59,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/cec3120b500945689aae21e5be9db024 2024-12-02T13:19:59,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/1efb1d6c3dea456fab7c9539a7938265 is 50, key is test_row_0/C:col10/1733145596634/Put/seqid=0 2024-12-02T13:19:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:19:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742427_1603 (size=12151) 2024-12-02T13:20:00,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:20:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:20:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:20:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:00,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/1efb1d6c3dea456fab7c9539a7938265 2024-12-02T13:20:00,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/427efafc099d496883e363c37237ad52 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52 2024-12-02T13:20:00,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52, entries=200, sequenceid=170, filesize=38.8 K 2024-12-02T13:20:00,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/cec3120b500945689aae21e5be9db024 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024 2024-12-02T13:20:00,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T13:20:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/1efb1d6c3dea456fab7c9539a7938265 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265 2024-12-02T13:20:00,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265, entries=150, sequenceid=170, filesize=11.9 K 2024-12-02T13:20:00,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 454ba568ee8de3afe25b6c498761192b in 1657ms, sequenceid=170, compaction requested=true 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:00,354 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:00,354 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:00,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:00,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,355 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,355 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/18c8c6e508a449a49d123246b968e395, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=47.3 K 2024-12-02T13:20:00,355 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=138.1 K 2024-12-02T13:20:00,355 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52] 2024-12-02T13:20:00,355 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c8c6e508a449a49d123246b968e395, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733145592931 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d2dbfcc0e8c4776810741b969bf6943, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733145594425 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf9571fbcd645eba07bb6fc483c12b3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733145592931 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 94730f5cb6804cb494e9b8963a0d8022, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733145594430 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6226b530f4a4f1496a15ad87939400e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733145593273 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ae8d3292ba48fca32cbd3df3180704, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733145594430 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting cec3120b500945689aae21e5be9db024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:00,356 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 427efafc099d496883e363c37237ad52, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:00,361 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:00,362 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:00,362 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/2faae00b29d44fb19c342f6be9a06987 is 50, key is test_row_0/B:col10/1733145596634/Put/seqid=0 2024-12-02T13:20:00,366 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d98f8b0e33d8474d826ab9aace182674_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:00,368 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d98f8b0e33d8474d826ab9aace182674_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:00,368 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d98f8b0e33d8474d826ab9aace182674_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:00,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742428_1604 (size=12493) 2024-12-02T13:20:00,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742429_1605 (size=4469) 2024-12-02T13:20:00,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,463 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:00,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e9b670a7d00e4e5b908bfe55ca71c886_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145598712/Put/seqid=0 2024-12-02T13:20:00,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742430_1606 (size=12304) 2024-12-02T13:20:00,787 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:00,787 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1c570219ce4f4285a8c98f0325e5df00 is 175, key is test_row_0/A:col10/1733145596634/Put/seqid=0 2024-12-02T13:20:00,789 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/2faae00b29d44fb19c342f6be9a06987 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2faae00b29d44fb19c342f6be9a06987 2024-12-02T13:20:00,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742431_1607 (size=31447) 2024-12-02T13:20:00,793 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 2faae00b29d44fb19c342f6be9a06987(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:00,793 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:00,793 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=12, startTime=1733145600354; duration=0sec 2024-12-02T13:20:00,793 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:00,793 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:00,793 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:00,794 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:00,794 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:00,794 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:00,794 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/ff875763c66a4d9abd077adbe6209035, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.6 K 2024-12-02T13:20:00,794 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ff875763c66a4d9abd077adbe6209035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1733145594425 2024-12-02T13:20:00,794 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1c570219ce4f4285a8c98f0325e5df00 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00 2024-12-02T13:20:00,795 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bfeae4db244572b362f2e3d13bde84, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733145594430 2024-12-02T13:20:00,795 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1efb1d6c3dea456fab7c9539a7938265, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:00,802 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into 1c570219ce4f4285a8c98f0325e5df00(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:00,802 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:00,802 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=12, startTime=1733145600354; duration=0sec 2024-12-02T13:20:00,802 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:00,802 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:00,806 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:00,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f8f2e0bb46e2422ba49e5c4c5d8867ab is 50, key is test_row_0/C:col10/1733145596634/Put/seqid=0 2024-12-02T13:20:00,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742432_1608 (size=12493) 2024-12-02T13:20:00,813 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f8f2e0bb46e2422ba49e5c4c5d8867ab as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f8f2e0bb46e2422ba49e5c4c5d8867ab 2024-12-02T13:20:00,826 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into f8f2e0bb46e2422ba49e5c4c5d8867ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:00,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:00,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:00,826 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:00,826 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145600354; duration=0sec 2024-12-02T13:20:00,827 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:00,827 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145600844, max timestamp=1733145600850Num uniques 859; 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145600844, max timestamp=1733145600850Num uniques 859; 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145600844, max timestamp=1733145600850Num uniques 859; 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:00,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145660870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145660870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145660871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145660874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145660875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:00,891 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e9b670a7d00e4e5b908bfe55ca71c886_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9b670a7d00e4e5b908bfe55ca71c886_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/70efad3926b54bc29ad88e4382da4249, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:00,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/70efad3926b54bc29ad88e4382da4249 is 175, key is test_row_0/A:col10/1733145598712/Put/seqid=0 2024-12-02T13:20:00,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742433_1609 (size=31105) 2024-12-02T13:20:00,895 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/70efad3926b54bc29ad88e4382da4249 2024-12-02T13:20:00,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/802e550cad024cc2890a0b76ca020672 is 50, key is test_row_0/B:col10/1733145598712/Put/seqid=0 2024-12-02T13:20:00,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742434_1610 (size=12151) 2024-12-02T13:20:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:20:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145660984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145660984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145660984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145660984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145660985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:00,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145661193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145661193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145661193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145661193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145661194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,307 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/802e550cad024cc2890a0b76ca020672 2024-12-02T13:20:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f0923de277a047749e5a7b1c07a30620 is 50, key is test_row_0/C:col10/1733145598712/Put/seqid=0 2024-12-02T13:20:01,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742435_1611 (size=12151) 2024-12-02T13:20:01,316 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f0923de277a047749e5a7b1c07a30620 2024-12-02T13:20:01,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/70efad3926b54bc29ad88e4382da4249 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249 2024-12-02T13:20:01,322 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249, entries=150, sequenceid=179, filesize=30.4 K 2024-12-02T13:20:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/802e550cad024cc2890a0b76ca020672 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672 2024-12-02T13:20:01,326 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T13:20:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/f0923de277a047749e5a7b1c07a30620 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620 2024-12-02T13:20:01,330 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620, entries=150, sequenceid=179, filesize=11.9 K 2024-12-02T13:20:01,331 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 454ba568ee8de3afe25b6c498761192b in 868ms, sequenceid=179, compaction requested=false 2024-12-02T13:20:01,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:01,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:01,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-02T13:20:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-02T13:20:01,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-02T13:20:01,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5460 sec 2024-12-02T13:20:01,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.5500 sec 2024-12-02T13:20:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:01,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-02T13:20:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145600870, max timestamp=1733145601505Num uniques 1068; 2024-12-02T13:20:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:20:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145600870, max timestamp=1733145601505Num uniques 1068; 2024-12-02T13:20:01,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:20:01,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733145600870, max timestamp=1733145601505Num uniques 1068; 2024-12-02T13:20:01,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:01,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:01,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bd5fe402f8d64df5a4bfe777d5878cb3_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:01,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742436_1612 (size=12304) 2024-12-02T13:20:01,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145601519, max timestamp=1733145601525Num uniques 859; 2024-12-02T13:20:01,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:01,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145601519, max timestamp=1733145601525Num uniques 859; 2024-12-02T13:20:01,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145601519, max timestamp=1733145601525Num uniques 859; 2024-12-02T13:20:01,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:01,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145601557, max timestamp=1733145601571Num uniques 1068; 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145601557, max timestamp=1733145601571Num uniques 1068; 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733145601557, max timestamp=1733145601571Num uniques 1068; 2024-12-02T13:20:01,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-02T13:20:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145661588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145661589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145661589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145661592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145661593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145661703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145661703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145661704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145661704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145661704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145661914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145661914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145661914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145661914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145661914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:01,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:01,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bd5fe402f8d64df5a4bfe777d5878cb3_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bd5fe402f8d64df5a4bfe777d5878cb3_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:01,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1e38f0000b7a4b6c85230fdae7b71ff7, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:01,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1e38f0000b7a4b6c85230fdae7b71ff7 is 175, key is test_row_0/A:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:01,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742437_1613 (size=31105) 2024-12-02T13:20:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145662224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145662224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145662224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145662224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145662224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1e38f0000b7a4b6c85230fdae7b71ff7 2024-12-02T13:20:02,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/45109c8036404d35818ab15d2e9df364 is 50, key is test_row_0/B:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:02,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742438_1614 (size=12151) 2024-12-02T13:20:02,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/45109c8036404d35818ab15d2e9df364 2024-12-02T13:20:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145662734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145662734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145662734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145662734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145662744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:02,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:02,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/780af33baa174be3975527e823f746da is 50, key is test_row_0/C:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:02,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742439_1615 (size=12151) 2024-12-02T13:20:02,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T13:20:02,953 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:02,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:20:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-12-02T13:20:02,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:02,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:20:02,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:20:02,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:20:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:03,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T13:20:03,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:03,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/780af33baa174be3975527e823f746da 2024-12-02T13:20:03,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/1e38f0000b7a4b6c85230fdae7b71ff7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7 2024-12-02T13:20:03,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7, entries=150, sequenceid=221, filesize=30.4 K 2024-12-02T13:20:03,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/45109c8036404d35818ab15d2e9df364 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364 2024-12-02T13:20:03,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T13:20:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/780af33baa174be3975527e823f746da as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da 2024-12-02T13:20:03,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da, entries=150, sequenceid=221, filesize=11.9 K 2024-12-02T13:20:03,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for 454ba568ee8de3afe25b6c498761192b in 1659ms, sequenceid=221, compaction requested=true 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:03,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:03,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:03,167 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:03,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:03,167 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:03,167 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,167 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=91.5 K 2024-12-02T13:20:03,167 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,168 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7] 2024-12-02T13:20:03,168 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c570219ce4f4285a8c98f0325e5df00, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:03,168 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70efad3926b54bc29ad88e4382da4249, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145598707 2024-12-02T13:20:03,168 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e38f0000b7a4b6c85230fdae7b71ff7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:03,169 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:03,169 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:03,169 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,169 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2faae00b29d44fb19c342f6be9a06987, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.9 K 2024-12-02T13:20:03,170 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2faae00b29d44fb19c342f6be9a06987, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:03,170 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 802e550cad024cc2890a0b76ca020672, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145598707 2024-12-02T13:20:03,171 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 45109c8036404d35818ab15d2e9df364, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:03,178 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:03,179 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#519 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:03,180 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/959a7926acbb49499feacc7234e6869e is 50, key is test_row_0/B:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:03,185 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412029c325c97ca604451b993bd255e5b9a2f_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:03,187 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412029c325c97ca604451b993bd255e5b9a2f_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:03,187 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029c325c97ca604451b993bd255e5b9a2f_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:03,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742440_1616 (size=12595) 2024-12-02T13:20:03,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742441_1617 (size=4469) 2024-12-02T13:20:03,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-12-02T13:20:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,261 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-12-02T13:20:03,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f655353a3e9b4c60962b792803b831df_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145601588/Put/seqid=0 2024-12-02T13:20:03,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742442_1618 (size=12454) 2024-12-02T13:20:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:03,600 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/959a7926acbb49499feacc7234e6869e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/959a7926acbb49499feacc7234e6869e 2024-12-02T13:20:03,604 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 959a7926acbb49499feacc7234e6869e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:03,604 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:03,604 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145603167; duration=0sec 2024-12-02T13:20:03,604 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:03,604 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:03,604 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:03,605 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:03,605 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:03,605 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:03,605 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f8f2e0bb46e2422ba49e5c4c5d8867ab, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=35.9 K 2024-12-02T13:20:03,605 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f2e0bb46e2422ba49e5c4c5d8867ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733145596634 2024-12-02T13:20:03,606 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting f0923de277a047749e5a7b1c07a30620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733145598707 2024-12-02T13:20:03,606 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 780af33baa174be3975527e823f746da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:03,607 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#518 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:03,608 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/92c85097ec7c423cb2f2a37b2d05a3a6 is 175, key is test_row_0/A:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:03,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742443_1619 (size=31549) 2024-12-02T13:20:03,626 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:03,626 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/1151b70e18a2416c82bd0e6c7f59c1d7 is 50, key is test_row_0/C:col10/1733145600874/Put/seqid=0 2024-12-02T13:20:03,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742444_1620 (size=12595) 2024-12-02T13:20:03,630 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/92c85097ec7c423cb2f2a37b2d05a3a6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6 2024-12-02T13:20:03,635 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into 92c85097ec7c423cb2f2a37b2d05a3a6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:03,635 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:03,635 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145603167; duration=0sec 2024-12-02T13:20:03,635 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/1151b70e18a2416c82bd0e6c7f59c1d7 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1151b70e18a2416c82bd0e6c7f59c1d7 2024-12-02T13:20:03,635 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:03,635 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:03,638 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into 1151b70e18a2416c82bd0e6c7f59c1d7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:03,638 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:03,638 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145603167; duration=0sec 2024-12-02T13:20:03,638 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:03,638 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:03,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:03,675 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f655353a3e9b4c60962b792803b831df_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f655353a3e9b4c60962b792803b831df_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:03,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/78e6488dc0664aba9c376dc7eea744eb, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/78e6488dc0664aba9c376dc7eea744eb is 175, key is test_row_0/A:col10/1733145601588/Put/seqid=0 2024-12-02T13:20:03,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742445_1621 (size=31255) 2024-12-02T13:20:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:03,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:03,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145663752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145663754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145663754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145663754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,758 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:03,758 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145663763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,766 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145663864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145663864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145664074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145664074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:04,088 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/78e6488dc0664aba9c376dc7eea744eb 2024-12-02T13:20:04,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/31c00ac48be44d53b346019b562ac1f6 is 50, key is test_row_0/B:col10/1733145601588/Put/seqid=0 2024-12-02T13:20:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:04,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742446_1622 (size=12301) 2024-12-02T13:20:04,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145664384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:04,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145664384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:04,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:04,497 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/31c00ac48be44d53b346019b562ac1f6 2024-12-02T13:20:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c0261995fc5342a9a8d6aadcda9809ec is 50, key is test_row_0/C:col10/1733145601588/Put/seqid=0 2024-12-02T13:20:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742447_1623 (size=12301) 2024-12-02T13:20:04,506 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c0261995fc5342a9a8d6aadcda9809ec 2024-12-02T13:20:04,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/78e6488dc0664aba9c376dc7eea744eb as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb 2024-12-02T13:20:04,511 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb, entries=150, sequenceid=275, filesize=30.5 K 2024-12-02T13:20:04,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/31c00ac48be44d53b346019b562ac1f6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6 2024-12-02T13:20:04,515 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T13:20:04,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c0261995fc5342a9a8d6aadcda9809ec as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec 2024-12-02T13:20:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,518 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T13:20:04,519 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 454ba568ee8de3afe25b6c498761192b in 1258ms, sequenceid=275, compaction requested=false 2024-12-02T13:20:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-12-02T13:20:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-12-02T13:20:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-02T13:20:04,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-12-02T13:20:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.5680 sec 2024-12-02T13:20:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:04,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:04,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029238b71ed0be4b10959e6697810f6bf4_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:04,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742448_1624 (size=14994) 2024-12-02T13:20:04,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:04,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029238b71ed0be4b10959e6697810f6bf4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029238b71ed0be4b10959e6697810f6bf4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:04,933 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/643ee7f1e5984361ab811d19bc981c8f, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:04,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/643ee7f1e5984361ab811d19bc981c8f is 175, key is test_row_0/A:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:04,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742449_1625 (size=39945) 2024-12-02T13:20:04,957 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/643ee7f1e5984361ab811d19bc981c8f 2024-12-02T13:20:04,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/9f5740f485f24e5fbc2cb7e73789a761 is 50, key is test_row_0/B:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:04,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145604934, max timestamp=1733145604961Num uniques 859; 2024-12-02T13:20:04,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:04,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145604934, max timestamp=1733145604963Num uniques 859; 2024-12-02T13:20:04,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:04,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145604934, max timestamp=1733145604963Num uniques 859; 2024-12-02T13:20:04,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742450_1626 (size=9857) 2024-12-02T13:20:05,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145665012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145665016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-02T13:20:05,104 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:05,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:20:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-12-02T13:20:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:05,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:20:05,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:20:05,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:20:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145665123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145665123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:05,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-02T13:20:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145665334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145665334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/9f5740f485f24e5fbc2cb7e73789a761 2024-12-02T13:20:05,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d9c0a8d97507462487f12f1577c78081 is 50, key is test_row_0/C:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742451_1627 (size=9857) 2024-12-02T13:20:05,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-02T13:20:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:05,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-02T13:20:05,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145665644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145665644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-02T13:20:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145665773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145665773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145665773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:05,775 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_1' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,775 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,776 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:05,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d9c0a8d97507462487f12f1577c78081 2024-12-02T13:20:05,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/643ee7f1e5984361ab811d19bc981c8f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f 2024-12-02T13:20:05,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f, entries=200, sequenceid=288, filesize=39.0 K 2024-12-02T13:20:05,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/9f5740f485f24e5fbc2cb7e73789a761 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761 2024-12-02T13:20:05,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761, entries=100, sequenceid=288, filesize=9.6 K 2024-12-02T13:20:05,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d9c0a8d97507462487f12f1577c78081 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081 2024-12-02T13:20:05,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081, entries=100, sequenceid=288, filesize=9.6 K 2024-12-02T13:20:05,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 454ba568ee8de3afe25b6c498761192b in 907ms, sequenceid=288, compaction requested=true 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:05,804 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:05,804 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:05,805 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:05,805 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:05,805 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:05,805 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,805 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:05,805 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=100.3 K 2024-12-02T13:20:05,805 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,805 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,805 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f] 2024-12-02T13:20:05,805 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/959a7926acbb49499feacc7234e6869e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=33.9 K 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c85097ec7c423cb2f2a37b2d05a3a6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 959a7926acbb49499feacc7234e6869e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c00ac48be44d53b346019b562ac1f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733145601586 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e6488dc0664aba9c376dc7eea744eb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733145601586 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5740f485f24e5fbc2cb7e73789a761, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145604895 2024-12-02T13:20:05,806 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643ee7f1e5984361ab811d19bc981c8f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145603751 2024-12-02T13:20:05,810 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:05,812 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:05,812 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412028edafc026e344bed93ad61f5eaca4f25_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:05,812 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/60946c7d9ab74d12aac823385ae0d691 is 50, key is test_row_0/B:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:05,813 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412028edafc026e344bed93ad61f5eaca4f25_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:05,813 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028edafc026e344bed93ad61f5eaca4f25_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:05,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742452_1628 (size=12847) 2024-12-02T13:20:05,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742453_1629 (size=4469) 2024-12-02T13:20:05,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:05,871 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:05,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029d758615a20b4132a20ed130117cb040_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145605015/Put/seqid=0 2024-12-02T13:20:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742454_1630 (size=12454) 2024-12-02T13:20:05,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:05,900 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029d758615a20b4132a20ed130117cb040_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d758615a20b4132a20ed130117cb040_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/7f7eff64b4494251bf6edeaa79d0565b, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:05,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/7f7eff64b4494251bf6edeaa79d0565b is 175, key is test_row_0/A:col10/1733145605015/Put/seqid=0 2024-12-02T13:20:05,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742455_1631 (size=31255) 2024-12-02T13:20:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:06,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:06,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145666170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145666173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,222 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#528 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:06,223 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/575a4d8d8c7346f2a3ef3db36064e91f is 175, key is test_row_0/A:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:06,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742456_1632 (size=31908) 2024-12-02T13:20:06,226 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/60946c7d9ab74d12aac823385ae0d691 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/60946c7d9ab74d12aac823385ae0d691 2024-12-02T13:20:06,231 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 60946c7d9ab74d12aac823385ae0d691(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:06,231 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145605804; duration=0sec 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:06,231 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:06,231 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:06,232 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1151b70e18a2416c82bd0e6c7f59c1d7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=33.9 K 2024-12-02T13:20:06,232 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1151b70e18a2416c82bd0e6c7f59c1d7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733145600874 2024-12-02T13:20:06,233 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c0261995fc5342a9a8d6aadcda9809ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733145601586 2024-12-02T13:20:06,233 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c0a8d97507462487f12f1577c78081, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145604895 2024-12-02T13:20:06,238 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:06,238 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/e8fff83373114f758d72c937d0102859 is 50, key is test_row_0/C:col10/1733145604895/Put/seqid=0 2024-12-02T13:20:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:06,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742457_1633 (size=12847) 2024-12-02T13:20:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145666284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145666284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,305 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/7f7eff64b4494251bf6edeaa79d0565b 2024-12-02T13:20:06,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1b183458ce0e4f59a26173577dc0d06f is 50, key is test_row_0/B:col10/1733145605015/Put/seqid=0 2024-12-02T13:20:06,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742458_1634 (size=12301) 2024-12-02T13:20:06,318 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1b183458ce0e4f59a26173577dc0d06f 2024-12-02T13:20:06,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/aa9743a5453b4d48a483bc65ccdb8700 is 50, key is test_row_0/C:col10/1733145605015/Put/seqid=0 2024-12-02T13:20:06,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742459_1635 (size=12301) 2024-12-02T13:20:06,328 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/aa9743a5453b4d48a483bc65ccdb8700 2024-12-02T13:20:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/7f7eff64b4494251bf6edeaa79d0565b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b 2024-12-02T13:20:06,334 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b, entries=150, sequenceid=325, filesize=30.5 K 2024-12-02T13:20:06,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1b183458ce0e4f59a26173577dc0d06f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f 2024-12-02T13:20:06,338 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f, entries=150, sequenceid=325, filesize=12.0 K 2024-12-02T13:20:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/aa9743a5453b4d48a483bc65ccdb8700 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700 2024-12-02T13:20:06,341 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700, entries=150, sequenceid=325, filesize=12.0 K 2024-12-02T13:20:06,342 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 454ba568ee8de3afe25b6c498761192b in 471ms, sequenceid=325, compaction requested=false 2024-12-02T13:20:06,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:06,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:06,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-12-02T13:20:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-12-02T13:20:06,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-02T13:20:06,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2360 sec 2024-12-02T13:20:06,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.2400 sec 2024-12-02T13:20:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:06,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:06,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b84dd18449834f5893f71ef051b7c5c1_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:06,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742460_1636 (size=14994) 2024-12-02T13:20:06,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:06,507 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b84dd18449834f5893f71ef051b7c5c1_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b84dd18449834f5893f71ef051b7c5c1_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:06,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/b218236ab99e4b6580c0c056275a68c8, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:06,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/b218236ab99e4b6580c0c056275a68c8 is 175, key is test_row_0/A:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:06,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742461_1637 (size=39949) 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145606526, max timestamp=1733145606535Num uniques 859; 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145606526, max timestamp=1733145606535Num uniques 859; 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733145606526, max timestamp=1733145606535Num uniques 859; 2024-12-02T13:20:06,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T13:20:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145666562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145666566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,630 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/575a4d8d8c7346f2a3ef3db36064e91f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f 2024-12-02T13:20:06,635 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into 575a4d8d8c7346f2a3ef3db36064e91f(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:06,636 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:06,636 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145605804; duration=0sec 2024-12-02T13:20:06,636 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:06,636 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:06,651 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/e8fff83373114f758d72c937d0102859 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/e8fff83373114f758d72c937d0102859 2024-12-02T13:20:06,654 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into e8fff83373114f758d72c937d0102859(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:06,655 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:06,655 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145605804; duration=0sec 2024-12-02T13:20:06,655 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:06,655 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:06,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145666674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145666674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,801 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T13:20:06,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145666884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145666884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:06,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/b218236ab99e4b6580c0c056275a68c8 2024-12-02T13:20:06,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 is 50, key is test_row_0/B:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:06,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742462_1638 (size=12301) 2024-12-02T13:20:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145667194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145667194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T13:20:07,254 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:07,254 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:20:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-12-02T13:20:07,256 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:20:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T13:20:07,256 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:20:07,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:20:07,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 2024-12-02T13:20:07,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/75432b0b33d2406aae0209fcfd26ec48 is 50, key is test_row_0/C:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742463_1639 (size=12301) 2024-12-02T13:20:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T13:20:07,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-12-02T13:20:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-12-02T13:20:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T13:20:07,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145667704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:07,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145667704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:07,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-12-02T13:20:07,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:07,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:07,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/75432b0b33d2406aae0209fcfd26ec48 2024-12-02T13:20:07,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/b218236ab99e4b6580c0c056275a68c8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8 2024-12-02T13:20:07,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8, entries=200, sequenceid=338, filesize=39.0 K 2024-12-02T13:20:07,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 2024-12-02T13:20:07,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6, entries=150, sequenceid=338, filesize=12.0 K 2024-12-02T13:20:07,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/75432b0b33d2406aae0209fcfd26ec48 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48 2024-12-02T13:20:07,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48, entries=150, sequenceid=338, filesize=12.0 K 2024-12-02T13:20:07,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 454ba568ee8de3afe25b6c498761192b in 1249ms, sequenceid=338, compaction requested=true 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:07,744 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:07,744 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:07,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:07,745 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,745 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=100.7 K 2024-12-02T13:20:07,745 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8] 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:07,745 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 575a4d8d8c7346f2a3ef3db36064e91f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145601586 2024-12-02T13:20:07,745 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,745 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/60946c7d9ab74d12aac823385ae0d691, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.6 K 2024-12-02T13:20:07,746 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7eff64b4494251bf6edeaa79d0565b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733145605000 2024-12-02T13:20:07,746 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 60946c7d9ab74d12aac823385ae0d691, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145601586 2024-12-02T13:20:07,746 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b218236ab99e4b6580c0c056275a68c8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:07,746 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b183458ce0e4f59a26173577dc0d06f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733145605000 2024-12-02T13:20:07,746 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 30bc0e5a49fd4fb7a770bab1dfe0d0f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:07,749 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:07,751 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:07,751 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1c7baad9fcc04bc2a57e9ae6933be278 is 50, key is test_row_0/B:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:07,752 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202392ea50e58f6466eb13cfe4932296f01_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:07,754 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202392ea50e58f6466eb13cfe4932296f01_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:07,754 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202392ea50e58f6466eb13cfe4932296f01_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:07,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742465_1641 (size=4469) 2024-12-02T13:20:07,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742464_1640 (size=12949) 2024-12-02T13:20:07,766 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#536 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:07,766 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d0c4ecf8a75343fd9f656bd581496c32 is 175, key is test_row_0/A:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:07,768 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/1c7baad9fcc04bc2a57e9ae6933be278 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1c7baad9fcc04bc2a57e9ae6933be278 2024-12-02T13:20:07,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742466_1642 (size=31903) 2024-12-02T13:20:07,772 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into 1c7baad9fcc04bc2a57e9ae6933be278(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:07,772 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:07,772 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145607744; duration=0sec 2024-12-02T13:20:07,772 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d0c4ecf8a75343fd9f656bd581496c32 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32 2024-12-02T13:20:07,772 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:07,772 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:07,772 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:07,773 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:07,773 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:07,773 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,773 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/e8fff83373114f758d72c937d0102859, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.6 K 2024-12-02T13:20:07,774 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fff83373114f758d72c937d0102859, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733145601586 2024-12-02T13:20:07,774 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9743a5453b4d48a483bc65ccdb8700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1733145605000 2024-12-02T13:20:07,774 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 75432b0b33d2406aae0209fcfd26ec48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:07,778 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into d0c4ecf8a75343fd9f656bd581496c32(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:07,778 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:07,779 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145607744; duration=0sec 2024-12-02T13:20:07,779 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:07,779 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:07,787 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:07,787 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/91efaf59a8ba4a2ba2d3f849663e8916 is 50, key is test_row_0/C:col10/1733145606494/Put/seqid=0 2024-12-02T13:20:07,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742467_1643 (size=12949) 2024-12-02T13:20:07,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-12-02T13:20:07,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:07,866 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-02T13:20:07,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T13:20:07,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f2461bbc1be147a39172537d53011a29_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145606555/Put/seqid=0 2024-12-02T13:20:07,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742468_1644 (size=12454) 2024-12-02T13:20:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T13:20:08,194 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/91efaf59a8ba4a2ba2d3f849663e8916 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/91efaf59a8ba4a2ba2d3f849663e8916 2024-12-02T13:20:08,196 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into 91efaf59a8ba4a2ba2d3f849663e8916(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:08,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:08,196 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145607744; duration=0sec 2024-12-02T13:20:08,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:08,196 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:08,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,281 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f2461bbc1be147a39172537d53011a29_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2461bbc1be147a39172537d53011a29_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/cc36488039864dbca320efc888805744, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/cc36488039864dbca320efc888805744 is 175, key is test_row_0/A:col10/1733145606555/Put/seqid=0 2024-12-02T13:20:08,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742469_1645 (size=31255) 2024-12-02T13:20:08,285 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/cc36488039864dbca320efc888805744 2024-12-02T13:20:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b6c9d89417eb48c89169b1efba3ab767 is 50, key is test_row_0/B:col10/1733145606555/Put/seqid=0 2024-12-02T13:20:08,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742470_1646 (size=12301) 2024-12-02T13:20:08,293 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b6c9d89417eb48c89169b1efba3ab767 2024-12-02T13:20:08,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fff1c6a3aa8d48beb7c3707d75dc0548 is 50, key is test_row_0/C:col10/1733145606555/Put/seqid=0 2024-12-02T13:20:08,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742471_1647 (size=12301) 2024-12-02T13:20:08,302 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fff1c6a3aa8d48beb7c3707d75dc0548 2024-12-02T13:20:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/cc36488039864dbca320efc888805744 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744 2024-12-02T13:20:08,308 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744, entries=150, sequenceid=376, filesize=30.5 K 2024-12-02T13:20:08,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b6c9d89417eb48c89169b1efba3ab767 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767 2024-12-02T13:20:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,312 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767, entries=150, sequenceid=376, filesize=12.0 K 2024-12-02T13:20:08,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/fff1c6a3aa8d48beb7c3707d75dc0548 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548, entries=150, sequenceid=376, filesize=12.0 K 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=0 B/0 for 454ba568ee8de3afe25b6c498761192b in 450ms, sequenceid=376, compaction requested=false 2024-12-02T13:20:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0620 sec 2024-12-02T13:20:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.0660 sec 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,394 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:08,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-12-02T13:20:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T13:20:08,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:20:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:20:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:20:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-12-02T13:20:08,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:08,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 159 msec 2024-12-02T13:20:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,714 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-02T13:20:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-12-02T13:20:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,716 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T13:20:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T13:20:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:08,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:08,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e35122b33da74b9889cd7d3e3e9b3cd4_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:08,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145608809, max timestamp=1733145608833Num uniques 859; 2024-12-02T13:20:08,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:08,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145608809, max timestamp=1733145608833Num uniques 859; 2024-12-02T13:20:08,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:08,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733145608809, max timestamp=1733145608833Num uniques 859; 2024-12-02T13:20:08,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/06e3e18f2515:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T13:20:08,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742472_1648 (size=22618) 2024-12-02T13:20:08,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:08,850 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e35122b33da74b9889cd7d3e3e9b3cd4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e35122b33da74b9889cd7d3e3e9b3cd4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:08,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/045f6767533d4b6db1ea0b40140c4a68, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:08,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/045f6767533d4b6db1ea0b40140c4a68 is 175, key is test_row_0/A:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:08,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742473_1649 (size=66023) 2024-12-02T13:20:08,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/045f6767533d4b6db1ea0b40140c4a68 2024-12-02T13:20:08,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:08,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:08,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:08,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:08,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:08,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:08,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7b0a1a257d4346de9fbcde470c24b7e9 is 50, key is test_row_0/B:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:08,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742474_1650 (size=12301) 2024-12-02T13:20:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145668884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:08,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145668887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145668994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145669003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:09,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145669204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145669213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7b0a1a257d4346de9fbcde470c24b7e9 2024-12-02T13:20:09,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/9c45ad5409f84c26925b5c62a32bef30 is 50, key is test_row_0/C:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:09,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742475_1651 (size=12301) 2024-12-02T13:20:09,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:09,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145669514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145669524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/9c45ad5409f84c26925b5c62a32bef30 2024-12-02T13:20:09,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/045f6767533d4b6db1ea0b40140c4a68 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68 2024-12-02T13:20:09,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68, entries=350, sequenceid=388, filesize=64.5 K 2024-12-02T13:20:09,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7b0a1a257d4346de9fbcde470c24b7e9 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9 2024-12-02T13:20:09,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9, entries=150, sequenceid=388, filesize=12.0 K 2024-12-02T13:20:09,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/9c45ad5409f84c26925b5c62a32bef30 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30 2024-12-02T13:20:09,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30, entries=150, sequenceid=388, filesize=12.0 K 2024-12-02T13:20:09,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 454ba568ee8de3afe25b6c498761192b in 929ms, sequenceid=388, compaction requested=true 2024-12-02T13:20:09,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:09,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:09,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:09,705 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,705 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1c7baad9fcc04bc2a57e9ae6933be278, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.7 K 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:09,705 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7baad9fcc04bc2a57e9ae6933be278, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:09,705 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=126.2 K 2024-12-02T13:20:09,705 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68] 2024-12-02T13:20:09,705 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c9d89417eb48c89169b1efba3ab767, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733145606555 2024-12-02T13:20:09,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c4ecf8a75343fd9f656bd581496c32, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:09,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc36488039864dbca320efc888805744, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733145606555 2024-12-02T13:20:09,706 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0a1a257d4346de9fbcde470c24b7e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608756 2024-12-02T13:20:09,706 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 045f6767533d4b6db1ea0b40140c4a68, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608750 2024-12-02T13:20:09,710 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:09,710 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/aa1040edc25344ce8f7da3da3f532c50 is 50, key is test_row_0/B:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:09,711 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:09,714 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202bb058c8ca8424340aa2f577d927bf785_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:09,716 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202bb058c8ca8424340aa2f577d927bf785_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:09,716 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bb058c8ca8424340aa2f577d927bf785_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742477_1653 (size=4469) 2024-12-02T13:20:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742476_1652 (size=13051) 2024-12-02T13:20:09,723 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#546 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:09,724 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5010336564a8468886a408e700bb573b is 175, key is test_row_0/A:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:09,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742478_1654 (size=32005) 2024-12-02T13:20:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:09,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T13:20:09,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bcee21ad3dc4439fa5750d5a695dca2d_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145608878/Put/seqid=0 2024-12-02T13:20:09,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742479_1655 (size=14994) 2024-12-02T13:20:09,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:09,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bcee21ad3dc4439fa5750d5a695dca2d_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bcee21ad3dc4439fa5750d5a695dca2d_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:09,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d8cd16eeb216453db38718b6d9a5747e, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:09,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d8cd16eeb216453db38718b6d9a5747e is 175, key is test_row_0/A:col10/1733145608878/Put/seqid=0 2024-12-02T13:20:09,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742480_1656 (size=39949) 2024-12-02T13:20:09,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145669800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145669805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37108 deadline: 1733145669823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,828 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_2' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145669913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145669913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:09,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:09,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145670023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37140 deadline: 1733145670033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145670124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,125 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/aa1040edc25344ce8f7da3da3f532c50 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aa1040edc25344ce8f7da3da3f532c50 2024-12-02T13:20:10,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145670124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,129 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into aa1040edc25344ce8f7da3da3f532c50(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:10,129 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145609704; duration=0sec 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:10,129 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:10,129 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,129 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/91efaf59a8ba4a2ba2d3f849663e8916, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.7 K 2024-12-02T13:20:10,130 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 91efaf59a8ba4a2ba2d3f849663e8916, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733145606167 2024-12-02T13:20:10,130 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5010336564a8468886a408e700bb573b as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b 2024-12-02T13:20:10,130 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting fff1c6a3aa8d48beb7c3707d75dc0548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733145606555 2024-12-02T13:20:10,130 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c45ad5409f84c26925b5c62a32bef30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608756 2024-12-02T13:20:10,133 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into 5010336564a8468886a408e700bb573b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:10,133 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:10,133 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145609703; duration=0sec 2024-12-02T13:20:10,133 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:10,133 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:10,134 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:10,134 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c6c5714a96004199a76e0726a6138c78 is 50, key is test_row_0/C:col10/1733145608773/Put/seqid=0 2024-12-02T13:20:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742481_1657 (size=13051) 2024-12-02T13:20:10,139 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/c6c5714a96004199a76e0726a6138c78 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c6c5714a96004199a76e0726a6138c78 2024-12-02T13:20:10,142 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into c6c5714a96004199a76e0726a6138c78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:10,142 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:10,142 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145609704; duration=0sec 2024-12-02T13:20:10,142 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:10,142 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:10,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d8cd16eeb216453db38718b6d9a5747e 2024-12-02T13:20:10,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ff43198b75cc45bc9cdf9eecabea1f01 is 50, key is test_row_0/B:col10/1733145608878/Put/seqid=0 2024-12-02T13:20:10,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742482_1658 (size=12301) 2024-12-02T13:20:10,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145670433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145670434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,455 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-12-02T13:20:10,455 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:10,455 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:10,455 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,457 INFO [Thread-2516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2516. 2024-12-02T13:20:10,457 DEBUG [Thread-2516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:10,457 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 DEBUG [Thread-2516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:10,458 INFO [Thread-2520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2520. 2024-12-02T13:20:10,458 DEBUG [Thread-2520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:10,458 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 DEBUG [Thread-2520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:10,458 INFO [Thread-2522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2522. 2024-12-02T13:20:10,458 DEBUG [Thread-2522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:10,458 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 DEBUG [Thread-2522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:10,459 INFO [Thread-2518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2518. 2024-12-02T13:20:10,459 DEBUG [Thread-2518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:10,459 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,459 DEBUG [Thread-2518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:10,459 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:10,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ff43198b75cc45bc9cdf9eecabea1f01 2024-12-02T13:20:10,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/7cc9243f227c43e3bf479fba8c649709 is 50, key is test_row_0/C:col10/1733145608878/Put/seqid=0 2024-12-02T13:20:10,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742483_1659 (size=12301) 2024-12-02T13:20:10,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:10,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37090 deadline: 1733145670944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37102 deadline: 1733145670944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:11,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/7cc9243f227c43e3bf479fba8c649709 2024-12-02T13:20:11,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/d8cd16eeb216453db38718b6d9a5747e as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e 2024-12-02T13:20:11,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e, entries=200, sequenceid=426, filesize=39.0 K 2024-12-02T13:20:11,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/ff43198b75cc45bc9cdf9eecabea1f01 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01 2024-12-02T13:20:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T13:20:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37074 deadline: 1733145671034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:11,035 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 06e3e18f2515,45693,1733145428280 for 'test_row_0' in 454ba568ee8de3afe25b6c498761192b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T13:20:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 , the old value is region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T13:20:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., hostname=06e3e18f2515,45693,1733145428280, seqNum=5 because the exception is null or not the one we care about 2024-12-02T13:20:11,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01, entries=150, sequenceid=426, filesize=12.0 K 2024-12-02T13:20:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/7cc9243f227c43e3bf479fba8c649709 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709 2024-12-02T13:20:11,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709, entries=150, sequenceid=426, filesize=12.0 K 2024-12-02T13:20:11,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 454ba568ee8de3afe25b6c498761192b in 1257ms, sequenceid=426, compaction requested=false 2024-12-02T13:20:11,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45693 {}] regionserver.HRegion(8855): Flush requested on 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:11,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:11,044 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:11,044 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:11,044 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:11,045 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:11,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120214a09c2d01ab42df8708adc1f270d133_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:11,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742484_1660 (size=12454) 2024-12-02T13:20:11,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:11,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120214a09c2d01ab42df8708adc1f270d133_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120214a09c2d01ab42df8708adc1f270d133_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:11,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/0593e5626f5244eeb114ebcf0f1c6cc0, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:11,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/0593e5626f5244eeb114ebcf0f1c6cc0 is 175, key is test_row_0/A:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:11,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742485_1661 (size=31255) 2024-12-02T13:20:11,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/0593e5626f5244eeb114ebcf0f1c6cc0 2024-12-02T13:20:11,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b750a17f4743485c8cbe2de2104162dc is 50, key is test_row_0/B:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742486_1662 (size=12301) 2024-12-02T13:20:11,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:11,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:11,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:11,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:11,955 INFO [Thread-2509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2509. 2024-12-02T13:20:11,955 INFO [Thread-2511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2511. 2024-12-02T13:20:11,955 DEBUG [Thread-2509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:11,955 DEBUG [Thread-2511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:11,955 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,955 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,955 DEBUG [Thread-2509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,955 DEBUG [Thread-2511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:11,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:11,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:12,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b750a17f4743485c8cbe2de2104162dc 2024-12-02T13:20:12,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/8ccb11f30c104fe3bc532420d6c7ba06 is 50, key is test_row_0/C:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:12,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742487_1663 (size=12301) 2024-12-02T13:20:12,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:12,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. as already flushing 2024-12-02T13:20:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(58): pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4177): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T13:20:12,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/8ccb11f30c104fe3bc532420d6c7ba06 2024-12-02T13:20:12,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/0593e5626f5244eeb114ebcf0f1c6cc0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0 2024-12-02T13:20:12,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0, entries=150, sequenceid=439, filesize=30.5 K 2024-12-02T13:20:12,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/b750a17f4743485c8cbe2de2104162dc as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc 2024-12-02T13:20:12,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc, entries=150, sequenceid=439, filesize=12.0 K 2024-12-02T13:20:12,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/8ccb11f30c104fe3bc532420d6c7ba06 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06 2024-12-02T13:20:12,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06, entries=150, sequenceid=439, filesize=12.0 K 2024-12-02T13:20:12,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 454ba568ee8de3afe25b6c498761192b in 1658ms, sequenceid=439, compaction requested=true 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:12,702 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:12,702 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 454ba568ee8de3afe25b6c498761192b:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T13:20:12,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/B is initiating minor compaction (all files) 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/A is initiating minor compaction (all files) 2024-12-02T13:20:12,703 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/B in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,703 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/A in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,703 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aa1040edc25344ce8f7da3da3f532c50, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.8 K 2024-12-02T13:20:12,703 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=100.8 K 2024-12-02T13:20:12,703 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. files: [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0] 2024-12-02T13:20:12,703 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1040edc25344ce8f7da3da3f532c50, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608756 2024-12-02T13:20:12,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5010336564a8468886a408e700bb573b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608756 2024-12-02T13:20:12,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting ff43198b75cc45bc9cdf9eecabea1f01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733145608878 2024-12-02T13:20:12,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8cd16eeb216453db38718b6d9a5747e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733145608845 2024-12-02T13:20:12,704 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting b750a17f4743485c8cbe2de2104162dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733145609799 2024-12-02T13:20:12,704 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0593e5626f5244eeb114ebcf0f1c6cc0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733145609799 2024-12-02T13:20:12,710 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:12,710 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#B#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:12,710 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/aea45ce2e2e04b12add075c11c80923f is 50, key is test_row_0/B:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:12,713 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202345924b12c5246c68d26bb639f6c2834_454ba568ee8de3afe25b6c498761192b store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:12,716 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202345924b12c5246c68d26bb639f6c2834_454ba568ee8de3afe25b6c498761192b, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:12,716 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202345924b12c5246c68d26bb639f6c2834_454ba568ee8de3afe25b6c498761192b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:12,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742488_1664 (size=13153) 2024-12-02T13:20:12,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742489_1665 (size=4469) 2024-12-02T13:20:12,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:12,720 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:12,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:12,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c01d05956ca74e5bb8182a008890dc47_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_1/A:col10/1733145611954/Put/seqid=0 2024-12-02T13:20:12,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742490_1666 (size=9914) 2024-12-02T13:20:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:13,057 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-12-02T13:20:13,057 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:13,057 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:13,057 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:13,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:13,121 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#A#compaction#555 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:13,122 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/8007d925358d4565a372c41f22d54e21 is 175, key is test_row_0/A:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:13,125 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/aea45ce2e2e04b12add075c11c80923f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aea45ce2e2e04b12add075c11c80923f 2024-12-02T13:20:13,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742491_1667 (size=32107) 2024-12-02T13:20:13,130 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/B of 454ba568ee8de3afe25b6c498761192b into aea45ce2e2e04b12add075c11c80923f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:13,130 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:13,131 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/B, priority=13, startTime=1733145612702; duration=0sec 2024-12-02T13:20:13,131 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T13:20:13,131 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:B 2024-12-02T13:20:13,131 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T13:20:13,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:13,132 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T13:20:13,132 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1541): 454ba568ee8de3afe25b6c498761192b/C is initiating minor compaction (all files) 2024-12-02T13:20:13,132 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 454ba568ee8de3afe25b6c498761192b/C in TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:13,132 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c6c5714a96004199a76e0726a6138c78, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06] into tmpdir=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp, totalSize=36.8 K 2024-12-02T13:20:13,132 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c5714a96004199a76e0726a6138c78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733145608756 2024-12-02T13:20:13,133 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc9243f227c43e3bf479fba8c649709, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733145608878 2024-12-02T13:20:13,133 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccb11f30c104fe3bc532420d6c7ba06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733145609799 2024-12-02T13:20:13,134 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c01d05956ca74e5bb8182a008890dc47_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c01d05956ca74e5bb8182a008890dc47_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:13,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/483be7f621e64af8a09713ee06f5dcd0, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/483be7f621e64af8a09713ee06f5dcd0 is 175, key is test_row_1/A:col10/1733145611954/Put/seqid=0 2024-12-02T13:20:13,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742492_1668 (size=22561) 2024-12-02T13:20:13,140 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 454ba568ee8de3afe25b6c498761192b#C#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T13:20:13,140 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/0a5352b8e58c4b8b96874383de005a3a is 50, key is test_row_0/C:col10/1733145609799/Put/seqid=0 2024-12-02T13:20:13,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742493_1669 (size=13153) 2024-12-02T13:20:13,537 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/8007d925358d4565a372c41f22d54e21 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/8007d925358d4565a372c41f22d54e21 2024-12-02T13:20:13,540 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/483be7f621e64af8a09713ee06f5dcd0 2024-12-02T13:20:13,544 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/A of 454ba568ee8de3afe25b6c498761192b into 8007d925358d4565a372c41f22d54e21(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:13,544 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:13,544 INFO [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/A, priority=13, startTime=1733145612702; duration=0sec 2024-12-02T13:20:13,544 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:13,544 DEBUG [RS:0;06e3e18f2515:45693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:A 2024-12-02T13:20:13,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/dd7b223228884731a29952af1c64a3f3 is 50, key is test_row_1/B:col10/1733145611954/Put/seqid=0 2024-12-02T13:20:13,548 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/0a5352b8e58c4b8b96874383de005a3a as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/0a5352b8e58c4b8b96874383de005a3a 2024-12-02T13:20:13,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742494_1670 (size=9857) 2024-12-02T13:20:13,553 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 454ba568ee8de3afe25b6c498761192b/C of 454ba568ee8de3afe25b6c498761192b into 0a5352b8e58c4b8b96874383de005a3a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T13:20:13,553 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:13,553 INFO [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., storeName=454ba568ee8de3afe25b6c498761192b/C, priority=13, startTime=1733145612702; duration=0sec 2024-12-02T13:20:13,553 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T13:20:13,553 DEBUG [RS:0;06e3e18f2515:45693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 454ba568ee8de3afe25b6c498761192b:C 2024-12-02T13:20:13,953 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/dd7b223228884731a29952af1c64a3f3 2024-12-02T13:20:13,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/6feb818bc5ff420f9f132dda73df7fd6 is 50, key is test_row_1/C:col10/1733145611954/Put/seqid=0 2024-12-02T13:20:13,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742495_1671 (size=9857) 2024-12-02T13:20:14,369 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/6feb818bc5ff420f9f132dda73df7fd6 2024-12-02T13:20:14,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/483be7f621e64af8a09713ee06f5dcd0 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/483be7f621e64af8a09713ee06f5dcd0 2024-12-02T13:20:14,380 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/483be7f621e64af8a09713ee06f5dcd0, entries=100, sequenceid=444, filesize=22.0 K 2024-12-02T13:20:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/dd7b223228884731a29952af1c64a3f3 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/dd7b223228884731a29952af1c64a3f3 2024-12-02T13:20:14,386 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/dd7b223228884731a29952af1c64a3f3, entries=100, sequenceid=444, filesize=9.6 K 2024-12-02T13:20:14,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/6feb818bc5ff420f9f132dda73df7fd6 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/6feb818bc5ff420f9f132dda73df7fd6 2024-12-02T13:20:14,391 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/6feb818bc5ff420f9f132dda73df7fd6, entries=100, sequenceid=444, filesize=9.6 K 2024-12-02T13:20:14,392 INFO [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=6.71 KB/6870 for 454ba568ee8de3afe25b6c498761192b in 1672ms, sequenceid=444, compaction requested=false 2024-12-02T13:20:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2603): Flush status journal for 454ba568ee8de3afe25b6c498761192b: 2024-12-02T13:20:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:14,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/06e3e18f2515:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=181 2024-12-02T13:20:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=181 2024-12-02T13:20:14,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-12-02T13:20:14,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.6770 sec 2024-12-02T13:20:14,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 5.6820 sec 2024-12-02T13:20:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T13:20:16,916 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:19,845 INFO [Thread-2513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2513. 2024-12-02T13:20:19,845 DEBUG [Thread-2513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T13:20:19,845 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:19,845 DEBUG [Thread-2513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:19,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3525 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3510 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3529 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3564 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3522 2024-12-02T13:20:19,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T13:20:19,846 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:20:19,846 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:20:19,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:19,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:19,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:19,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-02T13:20:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:19,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145619850"}]},"ts":"1733145619850"} 2024-12-02T13:20:19,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T13:20:19,852 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T13:20:19,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T13:20:19,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, UNASSIGN}] 2024-12-02T13:20:19,855 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, UNASSIGN 2024-12-02T13:20:19,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=CLOSING, regionLocation=06e3e18f2515,45693,1733145428280 2024-12-02T13:20:19,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=184, ppid=183, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, UNASSIGN because future has completed 2024-12-02T13:20:19,858 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T13:20:19,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280}] 2024-12-02T13:20:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:20,011 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(122): Close 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1722): Closing 454ba568ee8de3afe25b6c498761192b, disabling compactions & flushes 2024-12-02T13:20:20,011 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. after waiting 0 ms 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:20,011 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(2902): Flushing 454ba568ee8de3afe25b6c498761192b 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T13:20:20,011 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=A 2024-12-02T13:20:20,012 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:20,012 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=B 2024-12-02T13:20:20,012 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:20,012 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 454ba568ee8de3afe25b6c498761192b, store=C 2024-12-02T13:20:20,012 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T13:20:20,018 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202471ecdb23b4a43c58dc069de491706f4_454ba568ee8de3afe25b6c498761192b is 50, key is test_row_0/A:col10/1733145613055/Put/seqid=0 2024-12-02T13:20:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742496_1672 (size=9914) 2024-12-02T13:20:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:20,424 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T13:20:20,434 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202471ecdb23b4a43c58dc069de491706f4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202471ecdb23b4a43c58dc069de491706f4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:20,435 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5d9f7f8727664c75ad3022433bc0663f, store: [table=TestAcidGuarantees family=A region=454ba568ee8de3afe25b6c498761192b] 2024-12-02T13:20:20,436 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5d9f7f8727664c75ad3022433bc0663f is 175, key is test_row_0/A:col10/1733145613055/Put/seqid=0 2024-12-02T13:20:20,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742497_1673 (size=22561) 2024-12-02T13:20:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:20,841 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5d9f7f8727664c75ad3022433bc0663f 2024-12-02T13:20:20,846 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d68289cc29c4bdea3cf042e83912855 is 50, key is test_row_0/B:col10/1733145613055/Put/seqid=0 2024-12-02T13:20:20,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742498_1674 (size=9857) 2024-12-02T13:20:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:21,250 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d68289cc29c4bdea3cf042e83912855 2024-12-02T13:20:21,261 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 is 50, key is test_row_0/C:col10/1733145613055/Put/seqid=0 2024-12-02T13:20:21,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742499_1675 (size=9857) 2024-12-02T13:20:21,665 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 2024-12-02T13:20:21,674 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/A/5d9f7f8727664c75ad3022433bc0663f as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5d9f7f8727664c75ad3022433bc0663f 2024-12-02T13:20:21,679 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5d9f7f8727664c75ad3022433bc0663f, entries=100, sequenceid=452, filesize=22.0 K 2024-12-02T13:20:21,680 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/B/7d68289cc29c4bdea3cf042e83912855 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d68289cc29c4bdea3cf042e83912855 2024-12-02T13:20:21,683 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d68289cc29c4bdea3cf042e83912855, entries=100, sequenceid=452, filesize=9.6 K 2024-12-02T13:20:21,684 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/.tmp/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 2024-12-02T13:20:21,688 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d82f1e01ea4b4e51a0ab44f5b6a57f10, entries=100, sequenceid=452, filesize=9.6 K 2024-12-02T13:20:21,689 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 454ba568ee8de3afe25b6c498761192b in 1678ms, sequenceid=452, compaction requested=true 2024-12-02T13:20:21,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0] to archive 2024-12-02T13:20:21,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:20:21,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d26b01171f36490b8adc84f0860dd7f7 2024-12-02T13:20:21,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/928c329bcdce4374aefa351734093f18 2024-12-02T13:20:21,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/c36c990e510e4f41a825b37d4542fec4 2024-12-02T13:20:21,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f06438a762a84801a97bfccb56042c43 2024-12-02T13:20:21,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/11eab79a151b423cbc89ad1d1e8182fb 2024-12-02T13:20:21,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/bcf9571fbcd645eba07bb6fc483c12b3 2024-12-02T13:20:21,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/a2dbd409b2544ea78e6493db4737fc18 2024-12-02T13:20:21,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/f6226b530f4a4f1496a15ad87939400e 2024-12-02T13:20:21,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/72ae8d3292ba48fca32cbd3df3180704 2024-12-02T13:20:21,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/427efafc099d496883e363c37237ad52 2024-12-02T13:20:21,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1c570219ce4f4285a8c98f0325e5df00 2024-12-02T13:20:21,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/70efad3926b54bc29ad88e4382da4249 2024-12-02T13:20:21,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/92c85097ec7c423cb2f2a37b2d05a3a6 2024-12-02T13:20:21,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/1e38f0000b7a4b6c85230fdae7b71ff7 2024-12-02T13:20:21,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/78e6488dc0664aba9c376dc7eea744eb 2024-12-02T13:20:21,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/643ee7f1e5984361ab811d19bc981c8f 2024-12-02T13:20:21,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/575a4d8d8c7346f2a3ef3db36064e91f 2024-12-02T13:20:21,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/7f7eff64b4494251bf6edeaa79d0565b 2024-12-02T13:20:21,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/b218236ab99e4b6580c0c056275a68c8 2024-12-02T13:20:21,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d0c4ecf8a75343fd9f656bd581496c32 2024-12-02T13:20:21,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/cc36488039864dbca320efc888805744 2024-12-02T13:20:21,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/045f6767533d4b6db1ea0b40140c4a68 2024-12-02T13:20:21,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5010336564a8468886a408e700bb573b 2024-12-02T13:20:21,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/d8cd16eeb216453db38718b6d9a5747e 2024-12-02T13:20:21,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/0593e5626f5244eeb114ebcf0f1c6cc0 2024-12-02T13:20:21,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/43e7179bf5c94a4f8a1a380bf02b2886, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/18c8c6e508a449a49d123246b968e395, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2faae00b29d44fb19c342f6be9a06987, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/959a7926acbb49499feacc7234e6869e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/60946c7d9ab74d12aac823385ae0d691, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1c7baad9fcc04bc2a57e9ae6933be278, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aa1040edc25344ce8f7da3da3f532c50, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc] to archive 2024-12-02T13:20:21,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:20:21,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/56fb9d6dab1f483fbab6330ec399cf14 2024-12-02T13:20:21,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/bb8e51cf316648a883bcfadb876e6c3e 2024-12-02T13:20:21,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/43e7179bf5c94a4f8a1a380bf02b2886 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/43e7179bf5c94a4f8a1a380bf02b2886 2024-12-02T13:20:21,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2f502b3b9b2940f6923babfb68af0de2 2024-12-02T13:20:21,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ebde2f6380bb41bf9ca6e60d63b67512 2024-12-02T13:20:21,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/18c8c6e508a449a49d123246b968e395 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/18c8c6e508a449a49d123246b968e395 2024-12-02T13:20:21,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d0265be9b03462d9e8da1d67f5b64c3 2024-12-02T13:20:21,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1d2dbfcc0e8c4776810741b969bf6943 2024-12-02T13:20:21,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/94730f5cb6804cb494e9b8963a0d8022 2024-12-02T13:20:21,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2faae00b29d44fb19c342f6be9a06987 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/2faae00b29d44fb19c342f6be9a06987 2024-12-02T13:20:21,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/cec3120b500945689aae21e5be9db024 2024-12-02T13:20:21,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/802e550cad024cc2890a0b76ca020672 2024-12-02T13:20:21,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/959a7926acbb49499feacc7234e6869e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/959a7926acbb49499feacc7234e6869e 2024-12-02T13:20:21,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/45109c8036404d35818ab15d2e9df364 2024-12-02T13:20:21,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/31c00ac48be44d53b346019b562ac1f6 2024-12-02T13:20:21,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/60946c7d9ab74d12aac823385ae0d691 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/60946c7d9ab74d12aac823385ae0d691 2024-12-02T13:20:21,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/9f5740f485f24e5fbc2cb7e73789a761 2024-12-02T13:20:21,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1b183458ce0e4f59a26173577dc0d06f 2024-12-02T13:20:21,740 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1c7baad9fcc04bc2a57e9ae6933be278 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/1c7baad9fcc04bc2a57e9ae6933be278 2024-12-02T13:20:21,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/30bc0e5a49fd4fb7a770bab1dfe0d0f6 2024-12-02T13:20:21,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b6c9d89417eb48c89169b1efba3ab767 2024-12-02T13:20:21,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aa1040edc25344ce8f7da3da3f532c50 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aa1040edc25344ce8f7da3da3f532c50 2024-12-02T13:20:21,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7b0a1a257d4346de9fbcde470c24b7e9 2024-12-02T13:20:21,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/ff43198b75cc45bc9cdf9eecabea1f01 2024-12-02T13:20:21,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/b750a17f4743485c8cbe2de2104162dc 2024-12-02T13:20:21,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0f492e28db0415b8fc82cf6540642df, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/ff875763c66a4d9abd077adbe6209035, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f8f2e0bb46e2422ba49e5c4c5d8867ab, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1151b70e18a2416c82bd0e6c7f59c1d7, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/e8fff83373114f758d72c937d0102859, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/91efaf59a8ba4a2ba2d3f849663e8916, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c6c5714a96004199a76e0726a6138c78, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06] to archive 2024-12-02T13:20:21,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T13:20:21,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f5be943de4f04258b86f2a8e4a356b35 2024-12-02T13:20:21,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fba4ccdb162c4c97bd7819379953b08b 2024-12-02T13:20:21,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c3b577583bea46148aafab9121e22940 2024-12-02T13:20:21,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0f492e28db0415b8fc82cf6540642df to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0f492e28db0415b8fc82cf6540642df 2024-12-02T13:20:21,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/a84db884e8034deebe526b86d6ee724e 2024-12-02T13:20:21,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f32d2f3b19e24c53b83f757d894a6dd8 2024-12-02T13:20:21,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/ff875763c66a4d9abd077adbe6209035 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/ff875763c66a4d9abd077adbe6209035 2024-12-02T13:20:21,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/2b32d0810aae466c9ba6c6e82150c4cc 2024-12-02T13:20:21,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/41bfeae4db244572b362f2e3d13bde84 2024-12-02T13:20:21,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f8f2e0bb46e2422ba49e5c4c5d8867ab to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f8f2e0bb46e2422ba49e5c4c5d8867ab 2024-12-02T13:20:21,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1efb1d6c3dea456fab7c9539a7938265 2024-12-02T13:20:21,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/f0923de277a047749e5a7b1c07a30620 2024-12-02T13:20:21,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1151b70e18a2416c82bd0e6c7f59c1d7 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/1151b70e18a2416c82bd0e6c7f59c1d7 2024-12-02T13:20:21,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/780af33baa174be3975527e823f746da 2024-12-02T13:20:21,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c0261995fc5342a9a8d6aadcda9809ec 2024-12-02T13:20:21,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/e8fff83373114f758d72c937d0102859 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/e8fff83373114f758d72c937d0102859 2024-12-02T13:20:21,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d9c0a8d97507462487f12f1577c78081 2024-12-02T13:20:21,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/aa9743a5453b4d48a483bc65ccdb8700 2024-12-02T13:20:21,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/91efaf59a8ba4a2ba2d3f849663e8916 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/91efaf59a8ba4a2ba2d3f849663e8916 2024-12-02T13:20:21,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/75432b0b33d2406aae0209fcfd26ec48 2024-12-02T13:20:21,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/fff1c6a3aa8d48beb7c3707d75dc0548 2024-12-02T13:20:21,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c6c5714a96004199a76e0726a6138c78 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/c6c5714a96004199a76e0726a6138c78 2024-12-02T13:20:21,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/9c45ad5409f84c26925b5c62a32bef30 2024-12-02T13:20:21,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/7cc9243f227c43e3bf479fba8c649709 2024-12-02T13:20:21,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/8ccb11f30c104fe3bc532420d6c7ba06 2024-12-02T13:20:21,771 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits/455.seqid, newMaxSeqId=455, maxSeqId=4 2024-12-02T13:20:21,772 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. 2024-12-02T13:20:21,772 DEBUG [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] regionserver.HRegion(1676): Region close journal for 454ba568ee8de3afe25b6c498761192b: Waiting for close lock at 1733145620011Running coprocessor pre-close hooks at 1733145620011Disabling compacts and flushes for region at 1733145620011Disabling writes for close at 1733145620011Obtaining lock to block concurrent updates at 1733145620011Preparing flush snapshotting stores in 454ba568ee8de3afe25b6c498761192b at 1733145620011Finished memstore snapshotting TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733145620012 (+1 ms)Flushing stores of TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b. at 1733145620012Flushing 454ba568ee8de3afe25b6c498761192b/A: creating writer at 1733145620012Flushing mob file 454ba568ee8de3afe25b6c498761192b/A: appending metadata at 1733145620018 (+6 ms)Flushing mob file 454ba568ee8de3afe25b6c498761192b/A: closing flushed file at 1733145620018Flushing 454ba568ee8de3afe25b6c498761192b/A: appending metadata at 1733145620435 (+417 ms)Flushing 454ba568ee8de3afe25b6c498761192b/A: closing flushed file at 1733145620435Flushing 454ba568ee8de3afe25b6c498761192b/B: creating writer at 1733145620844 (+409 ms)Flushing 454ba568ee8de3afe25b6c498761192b/B: appending metadata at 1733145620845 (+1 ms)Flushing 454ba568ee8de3afe25b6c498761192b/B: closing flushed file at 1733145620845Flushing 454ba568ee8de3afe25b6c498761192b/C: creating writer at 1733145621258 (+413 ms)Flushing 454ba568ee8de3afe25b6c498761192b/C: appending metadata at 1733145621261 (+3 ms)Flushing 454ba568ee8de3afe25b6c498761192b/C: closing flushed file at 1733145621261Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ccc693d: reopening flushed file at 1733145621673 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a4064c7: reopening flushed file at 1733145621679 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@295f3b6e: reopening flushed file at 1733145621683 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 454ba568ee8de3afe25b6c498761192b in 1678ms, sequenceid=452, compaction requested=true at 1733145621689 (+6 ms)Writing region close event to WAL at 1733145621769 (+80 ms)Running coprocessor post-close hooks at 1733145621772 (+3 ms)Closed at 1733145621772 2024-12-02T13:20:21,773 INFO [RS_CLOSE_REGION-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_REGION, pid=185}] handler.UnassignRegionHandler(157): Closed 454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:21,774 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=184 updating hbase:meta row=454ba568ee8de3afe25b6c498761192b, regionState=CLOSED 2024-12-02T13:20:21,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=185, ppid=184, state=RUNNABLE, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 because future has completed 2024-12-02T13:20:21,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=185, resume processing ppid=184 2024-12-02T13:20:21,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, ppid=184, state=SUCCESS, hasLock=false; CloseRegionProcedure 454ba568ee8de3afe25b6c498761192b, server=06e3e18f2515,45693,1733145428280 in 1.9170 sec 2024-12-02T13:20:21,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-12-02T13:20:21,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=454ba568ee8de3afe25b6c498761192b, UNASSIGN in 1.9220 sec 2024-12-02T13:20:21,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-12-02T13:20:21,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9260 sec 2024-12-02T13:20:21,780 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733145621780"}]},"ts":"1733145621780"} 2024-12-02T13:20:21,781 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T13:20:21,781 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T13:20:21,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9350 sec 2024-12-02T13:20:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-12-02T13:20:21,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:21,996 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-02T13:20:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,000 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=186, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-12-02T13:20:22,001 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=186, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,006 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,009 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C, FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits] 2024-12-02T13:20:22,012 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/483be7f621e64af8a09713ee06f5dcd0 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/483be7f621e64af8a09713ee06f5dcd0 2024-12-02T13:20:22,013 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5d9f7f8727664c75ad3022433bc0663f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/5d9f7f8727664c75ad3022433bc0663f 2024-12-02T13:20:22,014 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/8007d925358d4565a372c41f22d54e21 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/A/8007d925358d4565a372c41f22d54e21 2024-12-02T13:20:22,016 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d68289cc29c4bdea3cf042e83912855 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/7d68289cc29c4bdea3cf042e83912855 2024-12-02T13:20:22,018 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aea45ce2e2e04b12add075c11c80923f to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/aea45ce2e2e04b12add075c11c80923f 2024-12-02T13:20:22,019 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/dd7b223228884731a29952af1c64a3f3 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/B/dd7b223228884731a29952af1c64a3f3 2024-12-02T13:20:22,022 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/0a5352b8e58c4b8b96874383de005a3a to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/0a5352b8e58c4b8b96874383de005a3a 2024-12-02T13:20:22,023 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/6feb818bc5ff420f9f132dda73df7fd6 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/6feb818bc5ff420f9f132dda73df7fd6 2024-12-02T13:20:22,024 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/C/d82f1e01ea4b4e51a0ab44f5b6a57f10 2024-12-02T13:20:22,027 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits/455.seqid to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b/recovered.edits/455.seqid 2024-12-02T13:20:22,028 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/default/TestAcidGuarantees/454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,028 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T13:20:22,029 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:20:22,030 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T13:20:22,033 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020abfcfd95c674791a1911aa84ca1e247_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020abfcfd95c674791a1911aa84ca1e247_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,034 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021385387da15e4908970e471d4b81b98a_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021385387da15e4908970e471d4b81b98a_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,036 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120214a09c2d01ab42df8708adc1f270d133_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120214a09c2d01ab42df8708adc1f270d133_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,037 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021f1645bc6d434798a5debec5e1e222b7_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021f1645bc6d434798a5debec5e1e222b7_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,038 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202471ecdb23b4a43c58dc069de491706f4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202471ecdb23b4a43c58dc069de491706f4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,039 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b35cb5cbf764997a51a61c1e977cd2e_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026b35cb5cbf764997a51a61c1e977cd2e_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,040 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028328577cbfaa4442b623ae95d25c4448_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028328577cbfaa4442b623ae95d25c4448_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,041 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029238b71ed0be4b10959e6697810f6bf4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029238b71ed0be4b10959e6697810f6bf4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,042 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d758615a20b4132a20ed130117cb040_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d758615a20b4132a20ed130117cb040_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,043 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029e626db340824a37ba2db4c7da41ed97_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029e626db340824a37ba2db4c7da41ed97_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,044 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b84dd18449834f5893f71ef051b7c5c1_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b84dd18449834f5893f71ef051b7c5c1_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,045 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bcee21ad3dc4439fa5750d5a695dca2d_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bcee21ad3dc4439fa5750d5a695dca2d_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,046 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bd5fe402f8d64df5a4bfe777d5878cb3_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bd5fe402f8d64df5a4bfe777d5878cb3_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,047 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c01d05956ca74e5bb8182a008890dc47_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c01d05956ca74e5bb8182a008890dc47_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,048 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e35122b33da74b9889cd7d3e3e9b3cd4_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e35122b33da74b9889cd7d3e3e9b3cd4_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,049 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e76ed04815cc4b8e854e67e07349fc41_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e76ed04815cc4b8e854e67e07349fc41_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,050 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9b670a7d00e4e5b908bfe55ca71c886_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e9b670a7d00e4e5b908bfe55ca71c886_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,051 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2461bbc1be147a39172537d53011a29_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2461bbc1be147a39172537d53011a29_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,052 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f39c0cdee2b44bd183a18cc54c315261_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f39c0cdee2b44bd183a18cc54c315261_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f655353a3e9b4c60962b792803b831df_454ba568ee8de3afe25b6c498761192b to hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f655353a3e9b4c60962b792803b831df_454ba568ee8de3afe25b6c498761192b 2024-12-02T13:20:22,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T13:20:22,056 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=186, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,059 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T13:20:22,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T13:20:22,062 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=186, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,062 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T13:20:22,063 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733145622062"}]},"ts":"9223372036854775807"} 2024-12-02T13:20:22,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T13:20:22,065 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 454ba568ee8de3afe25b6c498761192b, NAME => 'TestAcidGuarantees,,1733145587309.454ba568ee8de3afe25b6c498761192b.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T13:20:22,065 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T13:20:22,065 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733145622065"}]},"ts":"9223372036854775807"} 2024-12-02T13:20:22,067 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T13:20:22,068 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=186, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T13:20:22,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-12-02T13:20:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=186 2024-12-02T13:20:22,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T13:20:22,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T13:20:22,118 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 247) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 452), ProcessCount=11 (was 11), AvailableMemoryMB=2253 (was 2993) 2024-12-02T13:20:22,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T13:20:22,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:20:22,118 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:20:22,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:22,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:22,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T13:20:22,118 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T13:20:22,118 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=900566320, stopped=false 2024-12-02T13:20:22,119 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=06e3e18f2515,35237,1733145427680 2024-12-02T13:20:22,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T13:20:22,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T13:20:22,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:20:22,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:20:22,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T13:20:22,121 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T13:20:22,121 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:20:22,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T13:20:22,121 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T13:20:22,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:22,122 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '06e3e18f2515,45693,1733145428280' ***** 2024-12-02T13:20:22,122 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T13:20:22,122 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T13:20:22,123 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(959): stopping server 06e3e18f2515,45693,1733145428280 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;06e3e18f2515:45693. 2024-12-02T13:20:22,123 DEBUG [RS:0;06e3e18f2515:45693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T13:20:22,123 DEBUG [RS:0;06e3e18f2515:45693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T13:20:22,123 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T13:20:22,124 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T13:20:22,124 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T13:20:22,125 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:22,125 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T13:20:22,125 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T13:20:22,125 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T13:20:22,125 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T13:20:22,125 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T13:20:22,125 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T13:20:22,146 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/info/90298947b3e4433d81c12e966a476e46 is 95, key is TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5./info:/1733145461748/DeleteFamily/seqid=0 2024-12-02T13:20:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742500_1676 (size=6514) 2024-12-02T13:20:22,183 INFO [regionserver/06e3e18f2515:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T13:20:22,325 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:22,525 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:22,550 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/info/90298947b3e4433d81c12e966a476e46 2024-12-02T13:20:22,578 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/ns/7f8bf3da4e0c41f2ae048a6ceeb63698 is 93, key is TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5./ns:/1733145461712/DeleteFamily/seqid=0 2024-12-02T13:20:22,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742501_1677 (size=5906) 2024-12-02T13:20:22,726 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:22,926 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:22,967 INFO [regionserver/06e3e18f2515:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T13:20:22,967 INFO [regionserver/06e3e18f2515:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T13:20:22,981 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/ns/7f8bf3da4e0c41f2ae048a6ceeb63698 2024-12-02T13:20:23,003 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/rep_barrier/130ed9c31b914d00b37c517d13394dd5 is 102, key is TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5./rep_barrier:/1733145461712/DeleteFamily/seqid=0 2024-12-02T13:20:23,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742502_1678 (size=6025) 2024-12-02T13:20:23,126 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:23,326 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:23,408 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/rep_barrier/130ed9c31b914d00b37c517d13394dd5 2024-12-02T13:20:23,435 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/table/1a79763aac944880ad54399cf9578a89 is 96, key is TestAcidGuarantees,,1733145432409.f8f9c71492391b16a7ca84f91364ebb5./table:/1733145461712/DeleteFamily/seqid=0 2024-12-02T13:20:23,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742503_1679 (size=5936) 2024-12-02T13:20:23,527 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:23,727 DEBUG [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T13:20:23,840 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/table/1a79763aac944880ad54399cf9578a89 2024-12-02T13:20:23,851 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/info/90298947b3e4433d81c12e966a476e46 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/info/90298947b3e4433d81c12e966a476e46 2024-12-02T13:20:23,856 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/info/90298947b3e4433d81c12e966a476e46, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T13:20:23,857 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/ns/7f8bf3da4e0c41f2ae048a6ceeb63698 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/ns/7f8bf3da4e0c41f2ae048a6ceeb63698 2024-12-02T13:20:23,861 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/ns/7f8bf3da4e0c41f2ae048a6ceeb63698, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T13:20:23,861 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/rep_barrier/130ed9c31b914d00b37c517d13394dd5 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/rep_barrier/130ed9c31b914d00b37c517d13394dd5 2024-12-02T13:20:23,865 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/rep_barrier/130ed9c31b914d00b37c517d13394dd5, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T13:20:23,866 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/.tmp/table/1a79763aac944880ad54399cf9578a89 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/table/1a79763aac944880ad54399cf9578a89 2024-12-02T13:20:23,868 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/table/1a79763aac944880ad54399cf9578a89, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T13:20:23,868 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false 2024-12-02T13:20:23,872 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T13:20:23,872 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T13:20:23,872 INFO [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T13:20:23,872 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733145622124Running coprocessor pre-close hooks at 1733145622125 (+1 ms)Disabling compacts and flushes for region at 1733145622125Disabling writes for close at 1733145622125Obtaining lock to block concurrent updates at 1733145622125Preparing flush snapshotting stores in 1588230740 at 1733145622125Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733145622125Flushing stores of hbase:meta,,1.1588230740 at 1733145622126 (+1 ms)Flushing 1588230740/info: creating writer at 1733145622126Flushing 1588230740/info: appending metadata at 1733145622145 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733145622145Flushing 1588230740/ns: creating writer at 1733145622561 (+416 ms)Flushing 1588230740/ns: appending metadata at 1733145622577 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733145622577Flushing 1588230740/rep_barrier: creating writer at 1733145622984 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733145623003 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733145623003Flushing 1588230740/table: creating writer at 1733145623417 (+414 ms)Flushing 1588230740/table: appending metadata at 1733145623435 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733145623435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1499c813: reopening flushed file at 1733145623849 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7edf6fb0: reopening flushed file at 1733145623856 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8a77d3: reopening flushed file at 1733145623861 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74771e39: reopening flushed file at 1733145623865 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1743ms, sequenceid=90, compaction requested=false at 1733145623869 (+4 ms)Writing region close event to WAL at 1733145623870 (+1 ms)Running coprocessor post-close hooks at 1733145623872 (+2 ms)Closed at 1733145623872 2024-12-02T13:20:23,872 DEBUG [RS_CLOSE_META-regionserver/06e3e18f2515:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T13:20:23,927 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(976): stopping server 06e3e18f2515,45693,1733145428280; all regions closed. 2024-12-02T13:20:23,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741834_1010 (size=25344) 2024-12-02T13:20:23,933 DEBUG [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/oldWALs 2024-12-02T13:20:23,933 INFO [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 06e3e18f2515%2C45693%2C1733145428280.meta:.meta(num 1733145432008) 2024-12-02T13:20:23,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741832_1008 (size=19298907) 2024-12-02T13:20:23,937 DEBUG [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/oldWALs 2024-12-02T13:20:23,937 INFO [RS:0;06e3e18f2515:45693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 06e3e18f2515%2C45693%2C1733145428280:(num 1733145431122) 2024-12-02T13:20:23,937 DEBUG [RS:0;06e3e18f2515:45693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T13:20:23,937 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T13:20:23,937 INFO [RS:0;06e3e18f2515:45693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T13:20:23,937 INFO [RS:0;06e3e18f2515:45693 {}] hbase.ChoreService(370): Chore service for: regionserver/06e3e18f2515:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T13:20:23,938 INFO [RS:0;06e3e18f2515:45693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T13:20:23,938 INFO [regionserver/06e3e18f2515:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T13:20:23,938 INFO [RS:0;06e3e18f2515:45693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45693 2024-12-02T13:20:23,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/06e3e18f2515,45693,1733145428280 2024-12-02T13:20:23,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T13:20:23,941 INFO [RS:0;06e3e18f2515:45693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T13:20:23,942 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [06e3e18f2515,45693,1733145428280] 2024-12-02T13:20:23,943 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/06e3e18f2515,45693,1733145428280 already deleted, retry=false 2024-12-02T13:20:23,943 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 06e3e18f2515,45693,1733145428280 expired; onlineServers=0 2024-12-02T13:20:23,943 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '06e3e18f2515,35237,1733145427680' ***** 2024-12-02T13:20:23,943 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T13:20:23,943 INFO [M:0;06e3e18f2515:35237 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T13:20:23,943 INFO [M:0;06e3e18f2515:35237 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T13:20:23,943 DEBUG [M:0;06e3e18f2515:35237 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T13:20:23,944 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T13:20:23,944 DEBUG [M:0;06e3e18f2515:35237 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T13:20:23,944 DEBUG [master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.large.0-1733145430768 {}] cleaner.HFileCleaner(306): Exit Thread[master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.large.0-1733145430768,5,FailOnTimeoutGroup] 2024-12-02T13:20:23,944 DEBUG [master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.small.0-1733145430773 {}] cleaner.HFileCleaner(306): Exit Thread[master/06e3e18f2515:0:becomeActiveMaster-HFileCleaner.small.0-1733145430773,5,FailOnTimeoutGroup] 2024-12-02T13:20:23,944 INFO [M:0;06e3e18f2515:35237 {}] hbase.ChoreService(370): Chore service for: master/06e3e18f2515:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T13:20:23,944 INFO [M:0;06e3e18f2515:35237 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T13:20:23,944 DEBUG [M:0;06e3e18f2515:35237 {}] master.HMaster(1795): Stopping service threads 2024-12-02T13:20:23,944 INFO [M:0;06e3e18f2515:35237 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T13:20:23,944 INFO [M:0;06e3e18f2515:35237 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T13:20:23,944 ERROR [M:0;06e3e18f2515:35237 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44625 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44625,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T13:20:23,945 INFO [M:0;06e3e18f2515:35237 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T13:20:23,945 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T13:20:23,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T13:20:23,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T13:20:23,945 DEBUG [M:0;06e3e18f2515:35237 {}] zookeeper.ZKUtil(347): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T13:20:23,945 WARN [M:0;06e3e18f2515:35237 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T13:20:23,946 INFO [M:0;06e3e18f2515:35237 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/.lastflushedseqids 2024-12-02T13:20:23,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742504_1680 (size=72) 2024-12-02T13:20:24,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T13:20:24,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45693-0x1009b13d81b0001, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T13:20:24,044 INFO [RS:0;06e3e18f2515:45693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T13:20:24,044 INFO [RS:0;06e3e18f2515:45693 {}] regionserver.HRegionServer(1031): Exiting; stopping=06e3e18f2515,45693,1733145428280; zookeeper connection closed. 2024-12-02T13:20:24,044 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@15f48b45 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@15f48b45 2024-12-02T13:20:24,045 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T13:20:24,355 INFO [M:0;06e3e18f2515:35237 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T13:20:24,355 INFO [M:0;06e3e18f2515:35237 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T13:20:24,355 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T13:20:24,355 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:20:24,356 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:20:24,356 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T13:20:24,356 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:20:24,356 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=787.49 KB heapSize=964.95 KB 2024-12-02T13:20:24,378 DEBUG [M:0;06e3e18f2515:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ebb992e2269d4ac1b43ac93c3d8ec79d is 82, key is hbase:meta,,1/info:regioninfo/1733145432094/Put/seqid=0 2024-12-02T13:20:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742505_1681 (size=5672) 2024-12-02T13:20:24,781 INFO [M:0;06e3e18f2515:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2183 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ebb992e2269d4ac1b43ac93c3d8ec79d 2024-12-02T13:20:24,798 DEBUG [M:0;06e3e18f2515:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a95ace9bcd33409584106a935cd065e9 is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1733145464917/Put/seqid=0 2024-12-02T13:20:24,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742506_1682 (size=48218) 2024-12-02T13:20:25,202 INFO [M:0;06e3e18f2515:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=786.93 KB at sequenceid=2183 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a95ace9bcd33409584106a935cd065e9 2024-12-02T13:20:25,205 INFO [M:0;06e3e18f2515:35237 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a95ace9bcd33409584106a935cd065e9 2024-12-02T13:20:25,219 DEBUG [M:0;06e3e18f2515:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0bcd600694849dbb864197381bf35b8 is 69, key is 06e3e18f2515,45693,1733145428280/rs:state/1733145430889/Put/seqid=0 2024-12-02T13:20:25,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073742507_1683 (size=5156) 2024-12-02T13:20:25,623 INFO [M:0;06e3e18f2515:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2183 (bloomFilter=true), to=hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0bcd600694849dbb864197381bf35b8 2024-12-02T13:20:25,634 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ebb992e2269d4ac1b43ac93c3d8ec79d as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ebb992e2269d4ac1b43ac93c3d8ec79d 2024-12-02T13:20:25,638 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ebb992e2269d4ac1b43ac93c3d8ec79d, entries=8, sequenceid=2183, filesize=5.5 K 2024-12-02T13:20:25,639 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a95ace9bcd33409584106a935cd065e9 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a95ace9bcd33409584106a935cd065e9 2024-12-02T13:20:25,643 INFO [M:0;06e3e18f2515:35237 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a95ace9bcd33409584106a935cd065e9 2024-12-02T13:20:25,643 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a95ace9bcd33409584106a935cd065e9, entries=186, sequenceid=2183, filesize=47.1 K 2024-12-02T13:20:25,643 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f0bcd600694849dbb864197381bf35b8 as hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0bcd600694849dbb864197381bf35b8 2024-12-02T13:20:25,646 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44625/user/jenkins/test-data/c04fc7f7-cdd6-d50e-c53b-0b225c9ab22b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f0bcd600694849dbb864197381bf35b8, entries=1, sequenceid=2183, filesize=5.0 K 2024-12-02T13:20:25,647 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(3140): Finished flush of dataSize ~787.49 KB/806389, heapSize ~964.66 KB/987808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2183, compaction requested=false 2024-12-02T13:20:25,649 INFO [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T13:20:25,649 DEBUG [M:0;06e3e18f2515:35237 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733145624355Disabling compacts and flushes for region at 1733145624355Disabling writes for close at 1733145624356 (+1 ms)Obtaining lock to block concurrent updates at 1733145624356Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733145624356Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=806389, getHeapSize=988048, getOffHeapSize=0, getCellsCount=2394 at 1733145624356Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733145624357 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733145624357Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733145624377 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733145624377Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733145624784 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733145624798 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733145624798Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733145625206 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733145625218 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733145625218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d312f33: reopening flushed file at 1733145625632 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f6ce32d: reopening flushed file at 1733145625639 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@168ac852: reopening flushed file at 1733145625643 (+4 ms)Finished flush of dataSize ~787.49 KB/806389, heapSize ~964.66 KB/987808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2183, compaction requested=false at 1733145625647 (+4 ms)Writing region close event to WAL at 1733145625649 (+2 ms)Closed at 1733145625649 2024-12-02T13:20:25,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37517 is added to blk_1073741830_1006 (size=949547) 2024-12-02T13:20:25,652 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T13:20:25,652 INFO [M:0;06e3e18f2515:35237 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T13:20:25,652 INFO [M:0;06e3e18f2515:35237 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35237 2024-12-02T13:20:25,652 INFO [M:0;06e3e18f2515:35237 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T13:20:25,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T13:20:25,754 INFO [M:0;06e3e18f2515:35237 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T13:20:25,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x1009b13d81b0000, quorum=127.0.0.1:60906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T13:20:25,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T13:20:25,767 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T13:20:25,767 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T13:20:25,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T13:20:25,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.log.dir/,STOPPED} 2024-12-02T13:20:25,771 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T13:20:25,771 WARN [BP-1878892068-172.17.0.2-1733145424858 heartbeating to localhost/127.0.0.1:44625 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T13:20:25,771 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T13:20:25,771 WARN [BP-1878892068-172.17.0.2-1733145424858 heartbeating to localhost/127.0.0.1:44625 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1878892068-172.17.0.2-1733145424858 (Datanode Uuid 34595c3c-85ad-4c82-bb2d-22582016a5ec) service to localhost/127.0.0.1:44625 2024-12-02T13:20:25,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data1/current/BP-1878892068-172.17.0.2-1733145424858 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T13:20:25,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/cluster_9c4737c0-9328-3be7-9615-e41a8e179cb6/data/data2/current/BP-1878892068-172.17.0.2-1733145424858 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T13:20:25,774 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T13:20:25,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T13:20:25,781 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T13:20:25,781 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T13:20:25,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T13:20:25,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/565536a3-9599-2197-851c-cb1f0b1e1076/hadoop.log.dir/,STOPPED} 2024-12-02T13:20:25,795 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T13:20:25,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down